Permalink
Browse files

Parse build files with Jython to avoid shelling out to Python

Summary: Modify BuildFileToJsonParser to use Jython via JSR-223 rather than creating a new process to parse each python build file. Cuts build file parsing time roughly in half and avoids buck depending on the system python.

Test Plan: ant clean compile test && buck clean && buck build buck && buck test --all
  • Loading branch information...
1 parent 4d8bcda commit 575931561c61e82e477eecb2f0ffe8400bf2fc39 @jimpurbrick jimpurbrick committed with bolinfest May 3, 2013
View
@@ -19,5 +19,7 @@
<classpathentry kind="lib" path="lib/hamcrest-core-1.3.jar"/>
<classpathentry kind="lib" path="lib/hamcrest-library-1.3.jar"/>
<classpathentry kind="lib" path="lib/sdklib.jar"/>
+ <classpathentry kind="lib" path="lib/jyson-1.0.2.jar"/>
+ <classpathentry kind="lib" path="lib/jython-standalone-2.5.3.jar"/>
<classpathentry kind="output" path="build/classes"/>
</classpath>
View
@@ -7,6 +7,7 @@ local.properties
# Compiled Python
*.pyc
+*py.class
# IntelliJ, based on http://devnet.jetbrains.net/docs/DOC-1186
.idea/dictionaries
@@ -10,6 +10,8 @@
<root url="jar://$PROJECT_DIR$/lib/ddmlib-r21.jar!/" />
<root url="jar://$PROJECT_DIR$/lib/sdklib.jar!/" />
<root url="jar://$PROJECT_DIR$/lib/guava-14.0.1.jar!/" />
+ <root url="jar://$PROJECT_DIR$/lib/jyson-1.0.2.jar!/" />
+ <root url="jar://$PROJECT_DIR$/lib/jython-standalone-2.5.3.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES>
View
@@ -132,5 +132,7 @@ ${BUCK_DIRECTORY}/lib/jackson-core-2.0.5.jar:\
${BUCK_DIRECTORY}/lib/jackson-databind-2.0.5.jar:\
${BUCK_DIRECTORY}/lib/jsr305.jar:\
${BUCK_DIRECTORY}/lib/sdklib.jar:\
-${BUCK_DIRECTORY}/lib/ddmlib-r21.jar \
+${BUCK_DIRECTORY}/lib/ddmlib-r21.jar:\
+${BUCK_DIRECTORY}/lib/jython-standalone-2.5.3.jar:\
+${BUCK_DIRECTORY}/lib/jyson-1.0.2.jar \
com.facebook.buck.cli.Main "$@"
View
@@ -101,6 +101,8 @@
<include name="hamcrest-core-1.3.jar" />
<include name="hamcrest-library-1.3.jar" />
<include name="objenesis-1.2.jar" />
+ <include name="jython-standalone-2.5.3.jar" />
+ <include name="jyson-1.0.2.jar" />
</fileset>
<pathelement location="${testclasses.dir}" />
<pathelement location="${test.dir}" />
View
@@ -145,3 +145,21 @@ prebuilt_jar(
'//src/com/facebook/buck/shell:shell',
]
)
+
+jython_visibility = [
+ '//src/com/facebook/buck/shell:shell',
+ '//test/com/facebook/buck/cli:testutil',
+ '//test/com/facebook/buck/parser:parser',
+]
+
+prebuilt_jar(
+ name = 'jython',
+ binary_jar = 'jython-standalone-2.5.3.jar',
+ visibility = jython_visibility,
+)
+
+prebuilt_jar(
+ name = 'jyson',
+ binary_jar = 'jyson-1.0.2.jar',
+ visibility = jython_visibility,
+)
View
Binary file not shown.
Binary file not shown.
@@ -17,23 +17,33 @@
package com.facebook.buck.json;
import com.facebook.buck.util.Ansi;
-import com.facebook.buck.util.InputStreamConsumer;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
+import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.io.Closer;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.PipedReader;
+import java.io.PipedWriter;
import java.io.Reader;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import javax.script.ScriptException;
/**
* This is a special JSON parser that is customized to consume the JSON output of buck.py. In
@@ -50,6 +60,16 @@
private static final String PATH_TO_BUCK_PY = System.getProperty("buck.path_to_buck_py",
"src/com/facebook/buck/parser/buck.py");
+ private static final ExecutorService executor = Executors.newSingleThreadExecutor();
+ private static final ScriptEngine engine = new ScriptEngineManager().getEngineByName("python");
+ private static final String python = Joiner.on(System.getProperty("line.separator")).join(
+ "import sys",
+ "import os.path",
+ "sys.path.append(os.path.dirname(\"%s\"))",
+ "import buck",
+ "sys.argv=[\"%s\"]",
+ "buck.main()");
+
private final JsonParser parser;
public BuildFileToJsonParser(String json) throws JsonParseException, IOException {
@@ -143,6 +163,40 @@ public BuildFileToJsonParser(Reader json) throws JsonParseException, IOException
return getAllRules(rootPath.getAbsolutePath(), Optional.<String>absent(), includes, ansi);
}
+ private static class BuildFileRunner implements Runnable {
+
+ private final ImmutableList<String> args;
+ private PipedWriter outputWriter;
+
+ public BuildFileRunner(ImmutableList<String> args, PipedReader outputReader) throws IOException {
+ this.args = args;
+ this.outputWriter = new PipedWriter();
+ outputWriter.connect(outputReader);
+ }
+
+ @Override
+ public void run() {
+ Closer closer = Closer.create();
+ try {
+ closer.register(outputWriter);
+
+ // TODO(user): call buck.py directly rather than emulating the old command line interface?
+ // TODO(user): escape args? (they are currently file names, which shouldn't contain quotes)
+ engine.getContext().setWriter(outputWriter);
+ engine.eval(String.format(python, PATH_TO_BUCK_PY, Joiner.on("\",\"").join(args)));
+
+ } catch (ScriptException e) {
+ throw Throwables.propagate(e);
+ } finally {
+ try {
+ closer.close();
+ } catch (IOException e) {
+ Throwables.propagate(e);
+ }
+ }
+ }
+ }
+
/**
* @param rootPath Absolute path to the root of the project. buck.py uses this to determine the
* base path of the targets in the build file that it is parsing.
@@ -154,11 +208,27 @@ public BuildFileToJsonParser(Reader json) throws JsonParseException, IOException
Optional<String> buildFile,
Iterable<String> includes,
Ansi ansi) throws IOException {
- // A list of the build rules parsed from the build file.
+
+ // Run the build file in a background thread.
+ final ImmutableList<String> args = buildArgs(rootPath, buildFile, includes);
+ final PipedReader outputReader = new PipedReader();
+ BuildFileRunner runner = new BuildFileRunner(args, outputReader);
+ executor.execute(runner);
+
+ // Stream build rules from python.
+ BuildFileToJsonParser parser = new BuildFileToJsonParser(outputReader);
List<Map<String, Object>> rules = Lists.newArrayList();
+ Map<String, Object> value;
+ while ((value = parser.next()) != null) {
+ rules.add(value);
+ }
+
+ return rules;
+ }
+ private static ImmutableList<String> buildArgs(String rootPath, Optional<String> buildFile, Iterable<String> includes) {
// Create a process to run buck.py and read its stdout.
- List<String> args = Lists.newArrayList("python", PATH_TO_BUCK_PY, "--project_root", rootPath);
+ List<String> args = Lists.newArrayList("buck.py", "--project_root", rootPath);
// Add the --include flags.
for (String include : includes) {
@@ -170,42 +240,6 @@ public BuildFileToJsonParser(Reader json) throws JsonParseException, IOException
if (buildFile.isPresent()) {
args.add(buildFile.get());
}
-
- ProcessBuilder processBuilder = new ProcessBuilder(args);
- Process process = processBuilder.start();
- BuildFileToJsonParser parser = new BuildFileToJsonParser(process.getInputStream());
-
- // If the build file parses cleanly, then nothing should be written to standard error. We make
- // sure to consume stderr, just as we do in ShellCommand, to avoid potential deadlock.
- InputStreamConsumer stdErr = new InputStreamConsumer(
- process.getErrorStream(),
- System.err,
- true /* shouldPrintStdErr */,
- ansi);
- Thread stdErrConsumer = new Thread(stdErr);
- stdErrConsumer.start();
-
- // Read parsed JSON objects from the stream as they become available.
- Map<String, Object> value = null;
- while ((value = parser.next()) != null) {
- rules.add(value);
- }
-
- // Make sure the process exits with zero. If not, throw an exception. The error was likely
- // printed to stderr by the InputStreamConsumer.
- try {
- int exitCode = process.waitFor();
- if (exitCode != 0) {
- if (buildFile.isPresent()) {
- throw new RuntimeException("Parsing " + buildFile.get() + " did not exit cleanly");
- } else {
- throw new RuntimeException("Error parsing build files");
- }
- }
- } catch (InterruptedException e) {
- throw Throwables.propagate(e);
- }
-
- return rules;
+ return ImmutableList.copyOf(args);
}
}
@@ -1,11 +1,35 @@
-import argparse
+from __future__ import with_statement
+
+import optparse
import functools
-import json
import re
import os
import os.path
import posixpath
+try:
+ from com.xhaus.jyson import JysonCodec as json # jython embedded in buck
+except ImportError:
+ import json # python test case
+
+
+# TODO(user): upgrade to a jython including os.relpath
+def relpath(path, start=posixpath.curdir):
+ """
+ Return a relative filepath to path from the current directory or an optional start point.
+ """
+ if not path:
+ raise ValueError("no path specified")
+ start_list = posixpath.abspath(start).split(posixpath.sep)
+ path_list = posixpath.abspath(path).split(posixpath.sep)
+ # Work out how much of the filepath is shared by start and path.
+ common = len(posixpath.commonprefix([start_list, path_list]))
+ rel_list = [posixpath.pardir] * (len(start_list) - common) + path_list[common:]
+ if not rel_list:
+ return posixpath.curdir
+ return posixpath.join(*rel_list)
+
+
# When BUILD files are executed, the functions in this file tagged with
# @provide_for_build will be provided in the BUILD file's local symbol table.
#
@@ -103,7 +127,7 @@ def check_path(path):
for root, dirs, files in os.walk(search_base):
if len(files) == 0:
continue
- relative_root = os.path.relpath(root, search_base)
+ relative_root = relpath(root, search_base)
# The regexes generated by glob_pattern_to_regex_string don't
# expect a leading './'
if relative_root == '.':
@@ -588,19 +612,18 @@ def parse_git_ignore(gitignore_data):
# parser. That means that printing out other information for debugging purposes will likely break
# the JSON parsing, so be careful!
def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--project_root')
- parser.add_argument('--include', action='append')
- parser.add_argument('build_files', nargs='*')
- args = parser.parse_args()
+ parser = optparse.OptionParser()
+ parser.add_option('--project_root', action='store', type='string', dest='project_root')
+ parser.add_option('--include', action='append', dest='include')
+ (options, args) = parser.parse_args()
- project_root = args.project_root
+ project_root = options.project_root
len_suffix = -len('/' + BUILD_RULES_FILE_NAME)
build_files = None
- if args.build_files:
+ if args:
# The user has specified which build files to parse.
- build_files = args.build_files
+ build_files = args
else:
# Find all of the build files in the project root. Symlinks will not be traversed.
# Search must be done top-down so that directory filtering works as desired.
@@ -633,14 +656,14 @@ def main():
# or the files in includes through include_defs() don't pollute the namespace for
# subsequent build files.
build_env = {}
- relative_path_to_build_file = os.path.relpath(build_file, project_root)
+ relative_path_to_build_file = relpath(build_file, project_root)
build_env['BASE'] = relative_path_to_build_file[:len_suffix]
build_env['BUILD_FILE_DIRECTORY'] = os.path.dirname(build_file)
build_env['PROJECT_ROOT'] = project_root
build_env['BUILD_FILE_SYMBOL_TABLE'] = make_build_file_symbol_table(build_env)
# If there are any default includes, evaluate those first to populate the build_env.
- includes = args.include or []
+ includes = options.include or []
for include in includes:
include_defs(include, build_env)
execfile(os.path.join(project_root, build_file), build_env['BUILD_FILE_SYMBOL_TABLE'])
@@ -1,5 +1,6 @@
from buck import glob_pattern_to_regex_string
from buck import parse_git_ignore
+from buck import relpath
import unittest
import re
@@ -44,6 +45,7 @@ def test_glob_pattern_to_regex_string_double_star_no_subdir(self):
self.assertFalse(all_java_tests_re.match('com/example/Main.java'))
self.assertTrue(all_java_tests_re.match('com/example/MainTest.java'))
+
def test_git_ignore_parser(self):
# Test empty file
self.assertEqual([], parse_git_ignore([]))
@@ -74,5 +76,25 @@ def test_git_ignore_parser(self):
'/b/',
]))
+
+ # Test the temporary reimplementation of relpath
+ # TODO(user): upgrade to a jython including os.relpath
+ def test_relpath(self):
+ real_getcwd = os.getcwd
+ try:
+ os.getcwd = lambda: r"/home/user/bar"
+ curdir = os.path.split(os.getcwd())[-1]
+ self.assertRaises(ValueError, relpath, "")
+ self.assertEqual("a", relpath("a"))
+ self.assertEqual("a", relpath(posixpath.abspath("a")))
+ self.assertEqual("a/b", relpath("a/b"))
+ self.assertEqual("../a/b", relpath("../a/b"))
+ self.assertEqual("../" + curdir + "/a", relpath("a", "../b"))
+ self.assertEqual("../" + curdir + "/a/b", relpath("a/b", "../c"))
+ self.assertEqual("../../a", relpath("a", "b/c"))
+ finally:
+ os.getcwd = real_getcwd
+
+
if __name__ == '__main__':
unittest.main()
@@ -8,6 +8,8 @@ java_library(
deps = [
'//lib:guava',
'//lib:ddmlib',
+ '//lib:jyson',
+ '//lib:jython',
],
visibility = [
'PUBLIC',
@@ -8,6 +8,8 @@ java_test(
':NonCheckingBuildRuleFactoryParams',
'//lib:easymock',
'//lib:junit',
+ '//lib:jyson',
+ '//lib:jython',
'//src/com/facebook/buck/parser:parser',
'//test/com/facebook/buck/model:BuildTargetFactory',
'//test/com/facebook/buck/rules:testutil',

0 comments on commit 5759315

Please sign in to comment.