Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge branch 'master' of github.com:pantsbuild/pants into py2-wheels-…
…abi-specified
  • Loading branch information
Eric-Arellano committed Feb 22, 2019
2 parents 7da092b + 4097052 commit c9e1650
Show file tree
Hide file tree
Showing 13 changed files with 126 additions and 46 deletions.
8 changes: 4 additions & 4 deletions src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py
Expand Up @@ -401,7 +401,7 @@ def do_compile(self, invalidation_check, compile_contexts, classpath_product):
valid_targets = [vt.target for vt in invalidation_check.all_vts if vt.valid] valid_targets = [vt.target for vt in invalidation_check.all_vts if vt.valid]


if self.execution_strategy == self.HERMETIC: if self.execution_strategy == self.HERMETIC:
self._set_direcotry_digests_for_valid_target_classpath_directories(valid_targets, compile_contexts) self._set_directory_digests_for_valid_target_classpath_directories(valid_targets, compile_contexts)


for valid_target in valid_targets: for valid_target in valid_targets:
cc = self.select_runtime_context(compile_contexts[valid_target]) cc = self.select_runtime_context(compile_contexts[valid_target])
Expand Down Expand Up @@ -451,20 +451,20 @@ def _record_compile_classpath(self, classpath, target, outdir):
with open(path, 'w') as f: with open(path, 'w') as f:
f.write(text) f.write(text)


def _set_direcotry_digests_for_valid_target_classpath_directories(self, valid_targets, compile_contexts): def _set_directory_digests_for_valid_target_classpath_directories(self, valid_targets, compile_contexts):
snapshots = self.context._scheduler.capture_snapshots( snapshots = self.context._scheduler.capture_snapshots(
tuple(PathGlobsAndRoot(PathGlobs( tuple(PathGlobsAndRoot(PathGlobs(
[self._get_relative_classes_dir_from_target(target, compile_contexts)] [self._get_relative_classes_dir_from_target(target, compile_contexts)]
), get_buildroot()) for target in valid_targets)) ), get_buildroot()) for target in valid_targets))
[self._set_direcotry_digest_for_compile_context( [self._set_directory_digest_for_compile_context(
snapshot.directory_digest, target, compile_contexts) snapshot.directory_digest, target, compile_contexts)
for target, snapshot in list(zip(valid_targets, snapshots))] for target, snapshot in list(zip(valid_targets, snapshots))]


def _get_relative_classes_dir_from_target(self, target, compile_contexts): def _get_relative_classes_dir_from_target(self, target, compile_contexts):
cc = self.select_runtime_context(compile_contexts[target]) cc = self.select_runtime_context(compile_contexts[target])
return fast_relpath(cc.classes_dir.path, get_buildroot()) + '/**' return fast_relpath(cc.classes_dir.path, get_buildroot()) + '/**'


def _set_direcotry_digest_for_compile_context(self, directory_digest, target, compile_contexts): def _set_directory_digest_for_compile_context(self, directory_digest, target, compile_contexts):
cc = self.select_runtime_context(compile_contexts[target]) cc = self.select_runtime_context(compile_contexts[target])
new_classpath_entry = ClasspathEntry(cc.classes_dir.path, directory_digest) new_classpath_entry = ClasspathEntry(cc.classes_dir.path, directory_digest)
cc.classes_dir = new_classpath_entry cc.classes_dir = new_classpath_entry
Expand Down
7 changes: 7 additions & 0 deletions src/python/pants/engine/native.py
Expand Up @@ -376,6 +376,12 @@ def extern_store_i64(self, context_handle, i64):
c = self._ffi.from_handle(context_handle) c = self._ffi.from_handle(context_handle)
return c.to_value(i64) return c.to_value(i64)


@_extern_decl('Handle', ['ExternContext*', 'double'])
def extern_store_f64(self, context_handle, f64):
"""Given a context and double, return a new Handle to represent the double."""
c = self._ffi.from_handle(context_handle)
return c.to_value(f64)

@_extern_decl('Handle', ['ExternContext*', '_Bool']) @_extern_decl('Handle', ['ExternContext*', '_Bool'])
def extern_store_bool(self, context_handle, b): def extern_store_bool(self, context_handle, b):
"""Given a context and _Bool, return a new Handle to represent the _Bool.""" """Given a context and _Bool, return a new Handle to represent the _Bool."""
Expand Down Expand Up @@ -634,6 +640,7 @@ def init_externs():
self.ffi_lib.extern_store_bytes, self.ffi_lib.extern_store_bytes,
self.ffi_lib.extern_store_utf8, self.ffi_lib.extern_store_utf8,
self.ffi_lib.extern_store_i64, self.ffi_lib.extern_store_i64,
self.ffi_lib.extern_store_f64,
self.ffi_lib.extern_store_bool, self.ffi_lib.extern_store_bool,
self.ffi_lib.extern_project_ignoring_type, self.ffi_lib.extern_project_ignoring_type,
self.ffi_lib.extern_project_multi, self.ffi_lib.extern_project_multi,
Expand Down
27 changes: 27 additions & 0 deletions src/python/pants/notes/1.14.x.rst
Expand Up @@ -3,6 +3,33 @@
This document describes releases leading up to the ``1.14.x`` ``stable`` series. This document describes releases leading up to the ``1.14.x`` ``stable`` series.




1.14.0rc3 (2/21/2019)
---------------------

API Changes
~~~~~~~~~~~

* Add flags to processs_executor that say where to materialize output and what output is (#7201)
`PR #7201 <https://github.com/pantsbuild/pants/pull/7201>`_

* Resolve all platforms from all python targets (#7156)
`PR #7156 <https://github.com/pantsbuild/pants/pull/7156>`_

* Remove deprecated test classes (#7243)
`PR #7243 <https://github.com/pantsbuild/pants/pull/7243>`_

Bugfixes
~~~~~~~~

* Revert remote execution from tower to grpcio (#7256)
`PR #7256 <https://github.com/pantsbuild/pants/pull/7256>`_

* Avoid capturing Snapshots for previously digested codegen outputs (#7241)
`PR #7241 <https://github.com/pantsbuild/pants/pull/7241>`_

* Validate and maybe prune interpreter cache run over run (#7225)
`PR #7225 <https://github.com/pantsbuild/pants/pull/7225>`_

1.14.0rc2 (2/15/2019) 1.14.0rc2 (2/15/2019)
--------------------- ---------------------


Expand Down
19 changes: 18 additions & 1 deletion src/python/pants/util/objects.py
Expand Up @@ -132,13 +132,16 @@ def __eq__(self, other):
def __ne__(self, other): def __ne__(self, other):
return not (self == other) return not (self == other)


# NB: in Python 3, whenever __eq__ is overridden, __hash__() must also be
# explicitly implemented, otherwise Python will raise "unhashable type". See
# https://docs.python.org/3/reference/datamodel.html#object.__hash__.
def __hash__(self): def __hash__(self):
return super(DataType, self).__hash__() return super(DataType, self).__hash__()


# NB: As datatype is not iterable, we need to override both __iter__ and all of the # NB: As datatype is not iterable, we need to override both __iter__ and all of the
# namedtuple methods that expect self to be iterable. # namedtuple methods that expect self to be iterable.
def __iter__(self): def __iter__(self):
raise TypeError("'{}' object is not iterable".format(type(self).__name__)) raise self.make_type_error("datatype object is not iterable")


def _super_iter(self): def _super_iter(self):
return super(DataType, self).__iter__() return super(DataType, self).__iter__()
Expand Down Expand Up @@ -281,6 +284,20 @@ def __new__(cls, value):
""" """
return cls.create(value) return cls.create(value)


# TODO: figure out if this will always trigger on primitives like strings, and what situations
# won't call this __eq__ (and therefore won't raise like we want).
def __eq__(self, other):
"""Redefine equality to raise to nudge people to use static pattern matching."""
raise self.make_type_error(
"enum equality is defined to be an error -- use .resolve_for_enum_variant() instead!")
# Redefine the canary so datatype __new__ doesn't raise.
__eq__._eq_override_canary = None

# NB: as noted in datatype(), __hash__ must be explicitly implemented whenever __eq__ is
# overridden. See https://docs.python.org/3/reference/datamodel.html#object.__hash__.
def __hash__(self):
return super(ChoiceDatatype, self).__hash__()

@classmethod @classmethod
def create(cls, *args, **kwargs): def create(cls, *args, **kwargs):
"""Create an instance of this enum, using the default value if specified. """Create an instance of this enum, using the default value if specified.
Expand Down
2 changes: 1 addition & 1 deletion src/rust/engine/graph/src/entry.rs
Expand Up @@ -574,6 +574,6 @@ impl<N: Node> Entry<N> {
Some(Err(ref x)) => format!("{:?}", x), Some(Err(ref x)) => format!("{:?}", x),
None => "<None>".to_string(), None => "<None>".to_string(),
}; };
format!("{} == {}", self.node.content().format(), state).replace("\"", "\\\"") format!("{} == {}", self.node.content(), state).replace("\"", "\\\"")
} }
} }
14 changes: 8 additions & 6 deletions src/rust/engine/graph/src/lib.rs
Expand Up @@ -370,7 +370,7 @@ impl<N: Node> InnerGraph<N> {
let format = |eid: EntryId, depth: usize, is_last: bool| -> String { let format = |eid: EntryId, depth: usize, is_last: bool| -> String {
let entry = self.unsafe_entry_for_id(eid); let entry = self.unsafe_entry_for_id(eid);
let indent = " ".repeat(depth); let indent = " ".repeat(depth);
let output = format!("{}Computing {}", indent, entry.node().format()); let output = format!("{}Computing {}", indent, entry.node());
if is_last { if is_last {
format!( format!(
"{}\n{} {}", "{}\n{} {}",
Expand Down Expand Up @@ -430,7 +430,7 @@ impl<N: Node> InnerGraph<N> {


if deps.peek().is_none() { if deps.peek().is_none() {
// If the entry has no running deps, it is a leaf. Emit it. // If the entry has no running deps, it is a leaf. Emit it.
res.insert(self.unsafe_entry_for_id(id).node().format(), duration); res.insert(format!("{}", self.unsafe_entry_for_id(id).node()), duration);
if res.len() >= k { if res.len() >= k {
break; break;
} }
Expand Down Expand Up @@ -1055,10 +1055,6 @@ mod tests {
} }
} }


fn format(&self) -> String {
format!("{:?}", self)
}

fn digest(_result: Self::Item) -> Option<Digest> { fn digest(_result: Self::Item) -> Option<Digest> {
None None
} }
Expand All @@ -1068,6 +1064,12 @@ mod tests {
} }
} }


impl std::fmt::Display for TNode {
fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
write!(f, "{:?}", self)
}
}

impl TNode { impl TNode {
/// ///
/// Validates the given TNode output. Both node ids and context ids should increase left to /// Validates the given TNode output. Both node ids and context ids should increase left to
Expand Down
7 changes: 2 additions & 5 deletions src/rust/engine/graph/src/node.rs
@@ -1,7 +1,7 @@
// Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). // Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
// Licensed under the Apache License, Version 2.0 (see LICENSE). // Licensed under the Apache License, Version 2.0 (see LICENSE).


use std::fmt::Debug; use std::fmt::{Debug, Display};
use std::hash::Hash; use std::hash::Hash;


use boxfuture::BoxFuture; use boxfuture::BoxFuture;
Expand All @@ -21,17 +21,14 @@ pub type EntryId = stable_graph::NodeIndex<u32>;
/// ///
/// Note that it is assumed that Nodes are very cheap to clone. /// Note that it is assumed that Nodes are very cheap to clone.
/// ///
pub trait Node: Clone + Debug + Eq + Hash + Send + 'static { pub trait Node: Clone + Debug + Display + Eq + Hash + Send + 'static {
type Context: NodeContext<Node = Self>; type Context: NodeContext<Node = Self>;


type Item: Clone + Debug + Eq + Send + 'static; type Item: Clone + Debug + Eq + Send + 'static;
type Error: NodeError; type Error: NodeError;


fn run(self, context: Self::Context) -> BoxFuture<Self::Item, Self::Error>; fn run(self, context: Self::Context) -> BoxFuture<Self::Item, Self::Error>;


// TODO: Use a `Display` bound instead.
fn format(&self) -> String;

/// ///
/// If the given Node output represents an FS operation, returns its Digest. /// If the given Node output represents an FS operation, returns its Digest.
/// ///
Expand Down
8 changes: 8 additions & 0 deletions src/rust/engine/src/externs.rs
Expand Up @@ -120,6 +120,11 @@ pub fn store_i64(val: i64) -> Value {
with_externs(|e| (e.store_i64)(e.context, val).into()) with_externs(|e| (e.store_i64)(e.context, val).into())
} }


#[allow(dead_code)]
pub fn store_f64(val: f64) -> Value {
with_externs(|e| (e.store_f64)(e.context, val).into())
}

#[allow(dead_code)] #[allow(dead_code)]
pub fn store_bool(val: bool) -> Value { pub fn store_bool(val: bool) -> Value {
with_externs(|e| (e.store_bool)(e.context, val).into()) with_externs(|e| (e.store_bool)(e.context, val).into())
Expand Down Expand Up @@ -343,6 +348,7 @@ pub struct Externs {
pub store_bytes: StoreBytesExtern, pub store_bytes: StoreBytesExtern,
pub store_utf8: StoreUtf8Extern, pub store_utf8: StoreUtf8Extern,
pub store_i64: StoreI64Extern, pub store_i64: StoreI64Extern,
pub store_f64: StoreF64Extern,
pub store_bool: StoreBoolExtern, pub store_bool: StoreBoolExtern,
pub project_ignoring_type: ProjectIgnoringTypeExtern, pub project_ignoring_type: ProjectIgnoringTypeExtern,
pub project_multi: ProjectMultiExtern, pub project_multi: ProjectMultiExtern,
Expand Down Expand Up @@ -382,6 +388,8 @@ pub type StoreUtf8Extern = extern "C" fn(*const ExternContext, *const u8, u64) -


pub type StoreI64Extern = extern "C" fn(*const ExternContext, i64) -> Handle; pub type StoreI64Extern = extern "C" fn(*const ExternContext, i64) -> Handle;


pub type StoreF64Extern = extern "C" fn(*const ExternContext, f64) -> Handle;

pub type StoreBoolExtern = extern "C" fn(*const ExternContext, bool) -> Handle; pub type StoreBoolExtern = extern "C" fn(*const ExternContext, bool) -> Handle;


/// ///
Expand Down
6 changes: 4 additions & 2 deletions src/rust/engine/src/lib.rs
Expand Up @@ -62,8 +62,8 @@ use crate::externs::{
Buffer, BufferBuffer, CallExtern, CloneValExtern, CreateExceptionExtern, DropHandlesExtern, Buffer, BufferBuffer, CallExtern, CloneValExtern, CreateExceptionExtern, DropHandlesExtern,
EqualsExtern, EvalExtern, ExternContext, Externs, GeneratorSendExtern, HandleBuffer, EqualsExtern, EvalExtern, ExternContext, Externs, GeneratorSendExtern, HandleBuffer,
IdentifyExtern, LogExtern, ProjectIgnoringTypeExtern, ProjectMultiExtern, PyResult, IdentifyExtern, LogExtern, ProjectIgnoringTypeExtern, ProjectMultiExtern, PyResult,
SatisfiedByExtern, SatisfiedByTypeExtern, StoreBoolExtern, StoreBytesExtern, StoreI64Extern, SatisfiedByExtern, SatisfiedByTypeExtern, StoreBoolExtern, StoreBytesExtern, StoreF64Extern,
StoreTupleExtern, StoreUtf8Extern, TypeIdBuffer, TypeToStrExtern, ValToStrExtern, StoreI64Extern, StoreTupleExtern, StoreUtf8Extern, TypeIdBuffer, TypeToStrExtern, ValToStrExtern,
}; };
use crate::handles::Handle; use crate::handles::Handle;
use crate::rule_graph::{GraphMaker, RuleGraph}; use crate::rule_graph::{GraphMaker, RuleGraph};
Expand Down Expand Up @@ -119,6 +119,7 @@ pub extern "C" fn externs_set(
store_bytes: StoreBytesExtern, store_bytes: StoreBytesExtern,
store_utf8: StoreUtf8Extern, store_utf8: StoreUtf8Extern,
store_i64: StoreI64Extern, store_i64: StoreI64Extern,
store_f64: StoreF64Extern,
store_bool: StoreBoolExtern, store_bool: StoreBoolExtern,
project_ignoring_type: ProjectIgnoringTypeExtern, project_ignoring_type: ProjectIgnoringTypeExtern,
project_multi: ProjectMultiExtern, project_multi: ProjectMultiExtern,
Expand Down Expand Up @@ -146,6 +147,7 @@ pub extern "C" fn externs_set(
store_bytes, store_bytes,
store_utf8, store_utf8,
store_i64, store_i64,
store_f64,
store_bool, store_bool,
project_ignoring_type, project_ignoring_type,
project_multi, project_multi,
Expand Down
42 changes: 21 additions & 21 deletions src/rust/engine/src/nodes.rs
Expand Up @@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0 (see LICENSE). // Licensed under the Apache License, Version 2.0 (see LICENSE).


use std::collections::{BTreeMap, HashMap}; use std::collections::{BTreeMap, HashMap};
use std::fmt::Display;
use std::io::Write; use std::io::Write;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::Arc; use std::sync::Arc;
Expand Down Expand Up @@ -1061,27 +1062,6 @@ impl Node for NodeKey {
} }
} }


fn format(&self) -> String {
fn keystr(key: &Key) -> String {
externs::key_to_str(&key)
}
fn typstr(tc: &TypeConstraint) -> String {
externs::key_to_str(&tc.0)
}
// TODO: these should all be converted to fmt::Debug implementations, and then this method can
// go away in favor of the auto-derived Debug for this type.
match self {
&NodeKey::DigestFile(ref s) => format!("DigestFile({:?})", s.0),
&NodeKey::DownloadedFile(ref s) => format!("DownloadedFile({:?})", s.0),
&NodeKey::ExecuteProcess(ref s) => format!("ExecuteProcess({:?}", s.0),
&NodeKey::ReadLink(ref s) => format!("ReadLink({:?})", s.0),
&NodeKey::Scandir(ref s) => format!("Scandir({:?})", s.0),
&NodeKey::Select(ref s) => format!("Select({}, {})", s.params, typstr(&s.product)),
&NodeKey::Task(ref s) => format!("{:?}", s),
&NodeKey::Snapshot(ref s) => format!("Snapshot({})", keystr(&s.0)),
}
}

fn digest(res: NodeResult) -> Option<hashing::Digest> { fn digest(res: NodeResult) -> Option<hashing::Digest> {
match res { match res {
NodeResult::Digest(d) => Some(d), NodeResult::Digest(d) => Some(d),
Expand All @@ -1103,6 +1083,26 @@ impl Node for NodeKey {
} }
} }


impl Display for NodeKey {
fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
match self {
&NodeKey::DigestFile(ref s) => write!(f, "DigestFile({:?})", s.0),
&NodeKey::DownloadedFile(ref s) => write!(f, "DownloadedFile({:?})", s.0),
&NodeKey::ExecuteProcess(ref s) => write!(f, "ExecuteProcess({:?}", s.0),
&NodeKey::ReadLink(ref s) => write!(f, "ReadLink({:?})", s.0),
&NodeKey::Scandir(ref s) => write!(f, "Scandir({:?})", s.0),
&NodeKey::Select(ref s) => write!(
f,
"Select({}, {})",
s.params,
externs::key_to_str(&s.product.0)
),
&NodeKey::Task(ref s) => write!(f, "{:?}", s),
&NodeKey::Snapshot(ref s) => write!(f, "Snapshot({})", externs::key_to_str(&s.0)),
}
}
}

impl NodeError for Failure { impl NodeError for Failure {
fn invalidated() -> Failure { fn invalidated() -> Failure {
Failure::Invalidated Failure::Invalidated
Expand Down
7 changes: 2 additions & 5 deletions src/rust/engine/src/scheduler.rs
Expand Up @@ -13,7 +13,7 @@ use crate::context::{Context, Core};
use crate::core::{Failure, Params, TypeConstraint, Value}; use crate::core::{Failure, Params, TypeConstraint, Value};
use crate::nodes::{NodeKey, Select, Tracer, TryInto, Visualizer}; use crate::nodes::{NodeKey, Select, Tracer, TryInto, Visualizer};
use crate::selectors; use crate::selectors;
use graph::{EntryId, Graph, InvalidationResult, Node, NodeContext}; use graph::{EntryId, Graph, InvalidationResult, NodeContext};
use indexmap::IndexMap; use indexmap::IndexMap;
use log::{debug, info, warn}; use log::{debug, info, warn};
use parking_lot::Mutex; use parking_lot::Mutex;
Expand Down Expand Up @@ -210,10 +210,7 @@ impl Scheduler {
// Otherwise (if it is a success, some other type of Failure, or if we've run // Otherwise (if it is a success, some other type of Failure, or if we've run
// out of retries) recover to complete the join, which will cause the results to // out of retries) recover to complete the join, which will cause the results to
// propagate to the user. // propagate to the user.
debug!( debug!("Root {} completed.", NodeKey::Select(Box::new(root)));
"Root {} completed.",
NodeKey::Select(Box::new(root)).format()
);
Ok(other.map(|res| { Ok(other.map(|res| {
res res
.try_into() .try_into()
Expand Down
Expand Up @@ -183,7 +183,10 @@ def test_ctypes_third_party_integration(self, toolchain_variant):
# TODO(#6848): this fails when run with gcc on osx as it requires gcc's libstdc++.so.6.dylib to # TODO(#6848): this fails when run with gcc on osx as it requires gcc's libstdc++.so.6.dylib to
# be available on the runtime library path. # be available on the runtime library path.
attempt_pants_run = Platform.create().resolve_for_enum_variant({ attempt_pants_run = Platform.create().resolve_for_enum_variant({
'darwin': toolchain_variant != 'gnu', 'darwin': toolchain_variant.resolve_for_enum_variant({
'gnu': False,
'llvm': True,
}),
'linux': True, 'linux': True,
}) })
if attempt_pants_run: if attempt_pants_run:
Expand Down
20 changes: 20 additions & 0 deletions tests/python/pants_test/util/test_objects.py
Expand Up @@ -741,6 +741,26 @@ def test_enum_instance_creation_errors(self):
with self.assertRaisesRegexp(EnumVariantSelectionError, expected_rx_falsy_value): with self.assertRaisesRegexp(EnumVariantSelectionError, expected_rx_falsy_value):
SomeEnum('') SomeEnum('')


def test_enum_comparison_fails(self):
enum_instance = SomeEnum(1)
rx_str = re.escape("enum equality is defined to be an error")
with self.assertRaisesRegexp(TypeCheckError, rx_str):
enum_instance == enum_instance
with self.assertRaisesRegexp(TypeCheckError, rx_str):
enum_instance != enum_instance
# Test that comparison also fails against another type.
with self.assertRaisesRegexp(TypeCheckError, rx_str):
enum_instance == 1
with self.assertRaisesRegexp(TypeCheckError, rx_str):
1 == enum_instance

class StrEnum(enum(['a'])): pass
enum_instance = StrEnum('a')
with self.assertRaisesRegexp(TypeCheckError, rx_str):
enum_instance == 'a'
with self.assertRaisesRegexp(TypeCheckError, rx_str):
'a' == enum_instance

def test_enum_resolve_variant(self): def test_enum_resolve_variant(self):
one_enum_instance = SomeEnum(1) one_enum_instance = SomeEnum(1)
two_enum_instance = SomeEnum(2) two_enum_instance = SomeEnum(2)
Expand Down

0 comments on commit c9e1650

Please sign in to comment.