Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge remote branch 'c2c/master'

  • Loading branch information...
commit 86f5ceec04823be981877af755e9c195933209bc 2 parents 464fcbd + 1b654c7
Whit Morriss authored
28 CHANGELOG.txt
View
@@ -1,3 +1,31 @@
+2011-10-06 elemoine <eric.lemoine@camptocamp.com>
+
+ * make it possible to supply multiple file paths in the buildout "config" var
+
+2011-10-06 bbinet <bruno.binet@camptocamp.com>
+
+ * add --base-dir option which sets a base directory for root dirs
+
+2011-10-06 bbinet <bruno.binet@camptocamp.com>
+
+ * add support for a --list-only option which outputs the list
+ javascript files that would have been merged
+ * activate the concatenate feature by passing the concat option to
+ merger.run
+
+2011-03-30 bbinet <bruno.binet@camptocamp.com>
+
+ * fix trailing space bug in @requires/@include statements
+
+2011-03-22 fredj <frederic.junod@camptocamp.com>
+
+ * refactor the python loggers.
+
+2011-11-17 bbinet <bruno.binet@camptocamp.com>
+
+ * Add support for directories exclusion
+ * Add support for regexp files exclusion
+
2009-06-17 whit <whit@opengeo.org>
* Add implicit file inclusion to merge.py. In a config defines no
3  MANIFEST.in
View
@@ -0,0 +1,3 @@
+include setup.py
+include pavement.py
+include paver-minilib.zip
17 README.rst
View
@@ -96,12 +96,18 @@ Options:
-v, --verbose
print more info
+ -l, --list-only
+ Only list javascript files that would have been merged
+
-o OUTPUT_DIR, --output=OUTPUT_DIR
- Output directory for files jsbuild creates
+ Output directory for files jsbuild creates (defaults to current directory)
-r RESOURCE_DIR, --resource=RESOURCE_DIR
base directory for resource files (for interpolation)
+ -b ROOT_DIR, --base-dir=ROOT_DIR
+ base directory for root dirs (defaults to current directory)
+
-j SINGLE_FILE, --just=SINGLE_FILE
*New in 1.1*: Only create file for this section
@@ -137,7 +143,9 @@ A section is formatted in the following fashion::
core/api.js
exclude=
- 3rd/logger.js
+ 3rd/exclude/file.js
+ 3rd/exclude/dir
+ r:3rd/exclude/.*debug.js
#...
@@ -147,6 +155,9 @@ section will be forced to load *after* all the other files (in the
order listed).
The files list in the `exclude` section will not be imported.
+An exclude entry can be a relative path to a file or directory, or can be
+a python regular expression starting with pattern `r:`, see python `re` syntax:
+http://docs.python.org/library/re.html#regular-expression-syntax
The configuration allows for the interpolation of variables defined in
the config file. '%(resource-dir)s' may be subsituted for the value
@@ -245,7 +256,7 @@ Whit Morriss (whit at opengeo.org) repackaged these scripts as jstools
and Tim Schaub (tschaub at opengeo.org) did extensive reworking of tsort.
-.. [#] See 'virtualenv <http://pypi.python.org/pypi/virtualenv>'_ for
+.. [#] See `virtualenv <http://pypi.python.org/pypi/virtualenv>`_ for
more information about the python environment. You may activate
and deactivate this environment to add the installed scripts to
your path, localize python package installs and other niceties
7 jstools/bo.py
View
@@ -23,9 +23,12 @@ def __init__(self, buildout, name, options):
def install(self):
- self.merge = merge.Merger.from_fn(self.options.get('config'),
+ self.merge = merge.Merger.from_fn(tuple(self.options.get('config').split()),
output_dir=self.options.get('output-dir'),
+ root_dir=self.options.get('base-dir'),
defaults=self.defaults,
- printer=self.buildout._logger.info)
+ printer=self.buildout._logger)
files = self.merge.run(uncompressed=not self.compress, single=self.only)
return files
+
+ update = install
6 jstools/bo.txt
View
@@ -19,10 +19,12 @@ Variables
* compress -- True or False, minify output files
- * config -- path to jsbuild config file
+ * config -- path(s) to jsbuild config file(s)
* resource-dir -- not required, used for interpolation of jsbuild config
+ * base-dir -- not required, the base directory of root dirs
+
* only -- output concatenated file for only this section of the 'config'
* output -- (for use with only) write file to this name
@@ -36,6 +38,7 @@ Basic example::
recipe=JSTools:buildjs
resource-dir=${buildout:develop-dir}/${client:folder-name}
config=${build-js:resource-dir}/all.cfg
+ ${build-js:resource-dir}/app.cfg
output-dir=${build-js:resource-dir}/script
compress=True
...
@@ -48,6 +51,7 @@ jsbuild config file to build, 'output' defines the file created.
recipe=JSTools:buildjs
resource-dir=${buildout:develop-dir}/${client:folder-name}
config=${build-js:resource-dir}/all.cfg
+ ${build-js:resource-dir}/app.cfg
output-dir=${build-js:resource-dir}/script
only=OpenLayers.js
output=${build-js:output-dir}/OL-uncompressed.js
33 jstools/build.py
View
@@ -6,11 +6,13 @@
import sys
from jstools import DIST
from jstools.merge import Merger
-from jstools.utils import arg_parser, printer as printer_factory
+from jstools.utils import arg_parser
import pkg_resources
import optparse
import os
+import logging
+logger = logging.getLogger('jstools.build')
curdir = os.path.abspath(os.curdir)
usage = "usage: %prog [options] filename1.cfg [filename2.cfg...]"
@@ -25,8 +27,13 @@
action="store_true",
dest="verbose",
default=False)
+default_parser.add_option('-l', '--list-only',
+ help="Only list javascript files that would have been merged",
+ action="store_true",
+ dest="list_only",
+ default=False)
default_parser.add_option('-o', '--output',
- help="Output directory",
+ help="Output directory (defaults to current directory)",
action="store",
dest="output_dir",
default=curdir)
@@ -35,6 +42,11 @@
action="store",
dest="resource_dir",
default=curdir)
+default_parser.add_option('-b', '--base-dir',
+ help="base directory for root dirs (defaults to current directory)",
+ action="store",
+ dest="root_dir",
+ default=curdir)
default_parser.add_option('-j', '--just',
help="Only create file for this section",
action="store",
@@ -54,20 +66,27 @@
@arg_parser(default_parser)
def default_merge(args=None, options=None, parser=None):
- printer = printer_factory(options.verbose)
+ if options.verbose:
+ logging.basicConfig(level=logging.DEBUG, format="%(message)s")
+ else:
+ logging.basicConfig(level=logging.INFO, format="%(message)s")
+
if len(args) > 1:
filenames = args[1:]
else:
parser.error("You must provide at least one config filename")
merger = Merger.from_fn(filenames,
output_dir=options.output_dir,
+ root_dir=options.root_dir,
defaults={'resource-dir':options.resource_dir},
- printer=printer)
+ printer=logger)
out = merger.run(uncompressed=options.uncompress,
single=options.single_file,
- compressor=options.compressor)
- printer("Done:", 0)
- printer("\n".join(out), 0)
+ concatenate=options.concat,
+ compressor=options.compressor,
+ list_only=options.list_only)
+ logger.info("Done:")
+ logger.info("\n".join(out))
def build():
6 jstools/deps.py
View
@@ -17,7 +17,7 @@ class DepMap(ConfigParser):
template = dict(require='// @requires %s\n',
include='// @include %s\n')
- def __init__(self, defaults=None, printer=logger.info):
+ def __init__(self, defaults=None, printer=logger):
ConfigParser.__init__(self, defaults)
self.printer = printer
@@ -26,14 +26,14 @@ def alias_map(self):
return utils.SectionMap(self, 'alias')
@classmethod
- def from_resource(cls, resource_name, dist=DIST, defaults=None, printer=logger.info):
+ def from_resource(cls, resource_name, dist=DIST, defaults=None, printer=logger):
conf = pkg_resources.resource_stream(dist, resource_name)
dmap = cls(defaults=defaults, printer=printer)
dmap.readfp(conf)
return dmap
@classmethod
- def from_path(cls, path, defaults=None, printer=logger.info):
+ def from_path(cls, path, defaults=None, printer=logger):
dmap = cls(defaults=defaults, printer=printer)
if isinstance(path, basestring):
path = path,
127 jstools/merge.py
View
@@ -14,9 +14,9 @@
import re
-DEP_LINE = re.compile("^// @[include|requires]")
+DEP_LINE = re.compile("^// @[include|requires?]")
RE_INCLUDE = re.compile("@include (.*)\n")
-RE_REQUIRE = re.compile("@requires (.*)\n")
+RE_REQUIRE = re.compile("@requires? (.*)\n")
SUFFIX_JAVASCRIPT = ".js"
_marker = object()
@@ -28,17 +28,36 @@ class MissingImport(Exception):
"""Exception raised when a listed import is not found in the lib."""
+class Exclude(object):
+ def __init__(self, exclude):
+ self.exclude = exclude
+ self.pattern = None
+ if exclude.startswith('r:'):
+ self.pattern = re.compile(exclude[2:])
+
+ def __eq__(self, other):
+ if self.pattern is None:
+ if self.exclude == other:
+ return True
+ else:
+ # test other assuming self.exclude is a directory
+ return other.startswith(self.exclude + \
+ ('' if self.exclude[-1]=='/' else '/'))
+ else:
+ return self.pattern.match(other) is not None
+
+
class Merger(ConfigParser):
- def __init__(self, output_dir, defaults=None, printer=logger.info):
+ def __init__(self, output_dir=None, root_dir=None, defaults=None, printer=logger):
ConfigParser.__init__(self, defaults)
self.output_dir = output_dir
+ self.root_dir = root_dir
self.printer = printer
- self.reverse_included = dict()
@classmethod
- def from_fn(cls, fn, output_dir, defaults=None, printer=logger.info):
+ def from_fn(cls, fn, output_dir=None, root_dir=None, defaults=None, printer=logger):
"""Load up a list of config filenames in our merger"""
- merger = cls(output_dir, defaults=defaults, printer=printer)
+ merger = cls(output_dir, root_dir, defaults=defaults, printer=printer)
if isinstance(fn, basestring):
fn = fn,
fns = merger.read(fn)
@@ -46,23 +65,23 @@ def from_fn(cls, fn, output_dir, defaults=None, printer=logger.info):
return merger
@classmethod
- def from_resource(cls, resource_name, output_dir, requirement=REQ, defaults=None, printer=logger.info):
+ def from_resource(cls, resource_name, output_dir=None, root_dir=None, requirement=REQ, defaults=None, printer=logger):
conf = pkg_resources.resource_stream(requirement, resource_name)
- merger = cls(output_dir, defaults=defaults, printer=printer)
+ merger = cls(output_dir, root_dir, defaults=defaults, printer=printer)
merger.readfp(conf)
return merger
def make_sourcefile(self, sourcedir, filepath, exclude):
- self.printer("Importing: %s" % filepath)
- return SourceFile(sourcedir, filepath, exclude)
+ self.printer.debug("Importing: %s" % filepath)
+ return SourceFile(self.root_dir, sourcedir, filepath, exclude)
- def merge(self, cfg, depmap=None):
+ def extract_deps(self, cfg, depmap=None):
#@@ this function needs to be decomposed into smaller testable bits
sourcedirs = cfg['root']
# assemble all files in source directory according to config
include = cfg.get('include', tuple())
- exclude = cfg['exclude']
+ exclude = [Exclude(e) for e in cfg['exclude']]
all_inc = cfg['first'] + cfg['include'] + cfg['last']
files = {}
implicit = False
@@ -72,7 +91,7 @@ def merge(self, cfg, depmap=None):
for sourcedir in sourcedirs:
newfiles = []
- for filepath in jsfiles_for_dir(sourcedir):
+ for filepath in jsfiles_for_dir(os.path.join(self.root_dir, sourcedir)):
fitem = filepath, srcfile = filepath, self.make_sourcefile(sourcedir, filepath, exclude),
if implicit and not filepath in exclude:
all_inc.append(filepath)
@@ -87,10 +106,10 @@ def merge(self, cfg, depmap=None):
complete = True
for filepath, info in files.items():
for path in info.include + info.requires:
- if path not in cfg['exclude'] and not files.has_key(path):
+ if path not in exclude and not files.has_key(path):
complete = False
for sourcedir in sourcedirs:
- if os.path.exists(os.path.join(sourcedir, path)):
+ if os.path.exists(os.path.join(self.root_dir, sourcedir, path)):
files[path] = self.make_sourcefile(sourcedir, path, exclude)
break
else:
@@ -103,11 +122,11 @@ def merge(self, cfg, depmap=None):
# get tuple of files ordered by dependency
- self.printer("Sorting dependencies.")
+ self.printer.debug("Sorting dependencies.")
order = [x for x in tsort.sort(dependencies)]
# move forced first and last files to the required position
- self.printer("Re-ordering files.")
+ self.printer.debug("Re-ordering files.")
order = cfg['first'] + [item
for item in order
if ((item not in cfg['first']) and
@@ -119,24 +138,31 @@ def merge(self, cfg, depmap=None):
## Make sure all imports are in files dictionary
for part in parts:
for fp in cfg[part]:
- if not fp in cfg['exclude'] and not files.has_key(fp):
+ if not fp in exclude and not files.has_key(fp):
raise MissingImport("File from '%s' not found: %s" % (part, fp))
-
- ## Header inserted at the start of each file in the output
- HEADER = "/* " + "=" * 70 + "\n %s\n" + " " + "=" * 70 + " */\n\n"
## Output the files in the determined order
result = []
for fp in order:
- f = files[fp]
- self.printer("Exporting: " + f.filepath)
+ result.append(files[fp])
+
+ return result
+
+
+ def merge(self, cfg, depmap=None):
+ ## Header inserted at the start of each file in the output
+ HEADER = "/* " + "=" * 70 + "\n %s\n" + " " + "=" * 70 + " */\n\n"
+ result = []
+ files = self.extract_deps(cfg, depmap)
+ for f in files:
+ self.printer.debug("Exporting: " + f.filepath)
result.append(HEADER % f.filepath)
source = f.source
result.append(source)
if not source.endswith("\n"):
result.append("\n")
- self.printer("\nTotal files merged: %d " % len(files))
+ self.printer.debug("\nTotal files merged: %d " % len(files))
merged = "".join(result)
if cfg['closure']:
merged = '(function(){%s})();' % merged
@@ -160,7 +186,7 @@ def strip_deps(self, merged):
return "\n".join(x for x in merged.split('\n') if not DEP_LINE.match(x))
def compress(self, merged, plugin="default", cfg=None):
- self.printer("Compressing with %s" %plugin)
+ self.printer.debug("Compressing with %s" %plugin)
ep_map = pkg_resources.get_entry_map(DIST, "jstools.compressor")
args = None
func = ep_map.get(plugin).load()
@@ -168,7 +194,7 @@ def compress(self, merged, plugin="default", cfg=None):
def do_section(self, section, cfg):
header = "Building %s" % section
- self.printer("%s\n%s" % (header, "-" * len(header)))
+ self.printer.debug("%s\n%s" % (header, "-" * len(header)))
merged = self.merge(cfg)
if cfg.has_key('output'):
outputfilename = cfg['output']
@@ -221,22 +247,23 @@ def cat_run(self, outfile, sections, uncompressed=False, strip_deps=True, compre
seen = []
for name in newfiles:
print >> catted, cat[name]
- licout = lic[name]
- if licout not in seen: #slow?
- print >> license, licout
- seen.append(licout)
+ if name in lic:
+ licout = lic[name]
+ if licout not in seen: #slow?
+ print >> license, licout
+ seen.append(licout)
merged = catted.getvalue()
if not uncompressed:
- self.printer("Compressing %s" %outputfilename)
+ self.printer.debug("Compressing %s" %outputfilename)
merged = self.compress(merged, compressor, self)
elif strip_deps:
merged = self.strip_deps(merged)
merged_lic = license.getvalue()
merged = "%s\n%s" %(merged_lic, merged)
- self.printer("Writing to %s (%d KB).\n" % (outputfilename, int(len(merged) / 1024)))
+ self.printer.info("Writing to %s (%d KB)" % (outputfilename, int(len(merged) / 1024)))
sfb = file(outputfilename, "w").write(merged)
newfiles = [outputfilename]
@@ -251,25 +278,34 @@ def nocat_run(self, sections, uncompressed=False, strip_deps=True, compressor='d
outputfilename, merged = self.do_section(section, cfg)
if not uncompressed:
- self.printer("Compressing %s" %outputfilename)
+ self.printer.debug("Compressing %s" %outputfilename)
merged = self.compress(merged, compressor, self)
elif strip_deps:
merged = self.strip_deps(merged)
- self.printer("Writing to %s (%d KB).\n" % (outputfilename, int(len(merged) / 1024)))
+ self.printer.info("Writing to %s (%d KB)" % (outputfilename, int(len(merged) / 1024)))
if license:
- self.printer("Adding license file: %s" %cfg['license'])
+ self.printer.debug("Adding license file: %s" %cfg['license'])
merged = "\n".join((license, merged))
file(outputfilename, "w").write(merged)
newfiles.append(outputfilename)
return newfiles
- def run(self, uncompressed=False, single=None, strip_deps=True, concatenate=None, compressor="default"):
+ def list_run(self, sections):
+ newfiles = []
+ for section in sections:
+ cfg = self.make_cfg(section)
+ newfiles += [f.fullpath for f in self.extract_deps(cfg)]
+ return newfiles
+
+ def run(self, uncompressed=False, single=None, strip_deps=True, concatenate=None, compressor="default", list_only=False):
sections = self.js_sections()
if single is not None:
assert single in sections, ValueError("%s not in %s" %(single, sections))
sections = [single]
+ if list_only:
+ return self.list_run(sections)
if concatenate:
return self.cat_run(concatenate, sections, uncompressed, strip_deps, compressor)
else:
@@ -284,17 +320,28 @@ class SourceFile(object):
-- use depmap if given
"""
- def __init__(self, sourcedir, filepath, exclude, depmap=None):
+ def __init__(self, root_dir, sourcedir, filepath, exclude, depmap=None):
"""
"""
self.filepath = filepath
+ self.fullpath = os.path.join(sourcedir, filepath)
+ self.abspath = os.path.join(root_dir, self.fullpath)
self.exclude = exclude
- self.source = open(os.path.join(sourcedir, filepath), "U").read()
+ self._source = None
self._requires = _marker
self._include = _marker
self.depmap = depmap
@property
+ def source(self):
+ """
+ Provides lazy reading of the source file
+ """
+ if self._source is None:
+ self._source = open(self.abspath, "U").read()
+ return self._source
+
+ @property
def requires(self):
"""
Extracts the dependencies specified in the source code and returns
@@ -302,7 +349,7 @@ def requires(self):
"""
req = getattr(self, '_requires', None)
if req is _marker:
- self._requires = [x for x in RE_REQUIRE.findall(self.source)\
+ self._requires = [x.strip() for x in RE_REQUIRE.findall(self.source)\
if x not in self.exclude]
return self._requires
@@ -313,7 +360,7 @@ def include(self):
"""
req = getattr(self, '_include', None)
if req is _marker:
- self._include = [x for x in RE_INCLUDE.findall(self.source) \
+ self._include = [x.strip() for x in RE_INCLUDE.findall(self.source) \
if x not in self.exclude]
return self._include
1  jstools/proxy.py
View
@@ -1,7 +1,6 @@
from wsgiproxy.app import WSGIProxyApp
from cgi import parse_qs
-from urlparse import
import urlparse
import logging
import functools
25 jstools/utils.py
View
@@ -32,23 +32,6 @@ def caller(args=None, options=None, parser=optparser):
return caller
return wrapper
-
-def printer(verbosity):
- _print = lambda x: None
- logger = logging.getLogger('jstools')
- logger.setLevel(logging.DEBUG)
- ch = logging.StreamHandler()
- ch.setLevel(logging.DEBUG)
- ch.setFormatter(logging.Formatter("%(message)s"))
- logger.addHandler(ch)
- _print = logger.info
- def _printer(txt, threshold=1):
- if int(verbosity) >= threshold:
- _print(txt)
- return _printer
-
-
-
class SectionMap(DictMixin):
def __init__(self, cp, section):
if not cp.has_section(section):
@@ -105,11 +88,3 @@ def retrieve_config(section=None, strict=False):
if (user / fn).exists():
return section_or_parser(user / fn)
-
-
-
-
-
-
-
-
6 pavement.py
View
@@ -17,7 +17,7 @@
from paver import setuputils
setuputils.install_distutils_tasks()
-version = '0.1.5'
+version = '0.5'
try:
description = ''.join([x for x in open('README.rst')])
@@ -36,8 +36,8 @@
],
keywords='javascript',
author='assorted',
- author_email='info@opengeo.org',
- url='http://projects.opengeo.org/jstools',
+ author_email='jstools@googlegroups.com',
+ url='https://github.com/camptocamp/jstools',
license='various/BSDish',
packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
include_package_data=True,
3  setup.cfg
View
@@ -1,3 +0,0 @@
-[egg_info]
-tag_build = .dev
-tag_svn_revision = true
Please sign in to comment.
Something went wrong with that request. Please try again.