Skip to content

Commit

Permalink
Merge pull request #874 from andrasio/move-out-tag
Browse files Browse the repository at this point in the history 
Move out tags when parsing and building tree nodes.
  • Loading branch information
@andrasio
andrasio committed Oct 26, 2019
2 parents b7af343 + 2706ae0 commit 1b3a094
Show file tree
Hide file tree
Showing 4 changed files with 119 additions and 88 deletions.
183 changes: 107 additions & 76 deletions src/parser/parse/parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ pub fn leaf(input: NomSpan) -> IResult<NomSpan, TokenNode> {
}

#[tracable_parser]
pub fn token_list(input: NomSpan) -> IResult<NomSpan, Tagged<Vec<TokenNode>>> {
pub fn token_list(input: NomSpan) -> IResult<NomSpan, Spanned<Vec<TokenNode>>> {
let start = input.offset;
let (input, first) = node(input)?;

Expand All @@ -403,7 +403,7 @@ pub fn token_list(input: NomSpan) -> IResult<NomSpan, Tagged<Vec<TokenNode>>> {

Ok((
input,
make_token_list(first, list, None).tagged((start, end, None)),
make_token_list(first, list, None).spanned(Span::new(start, end)),
))
}

Expand Down Expand Up @@ -511,14 +511,14 @@ pub fn delimited_brace(input: NomSpan) -> IResult<NomSpan, TokenNode> {
}

#[tracable_parser]
pub fn raw_call(input: NomSpan) -> IResult<NomSpan, Tagged<CallNode>> {
pub fn raw_call(input: NomSpan) -> IResult<NomSpan, Spanned<CallNode>> {
let left = input.offset;
let (input, items) = token_list(input)?;
let right = input.offset;

Ok((
input,
TokenTreeBuilder::tagged_call(items.item, (left, right, input.extra)),
TokenTreeBuilder::spanned_call(items.item, Span::new(left, right)),
))
}

Expand Down Expand Up @@ -598,7 +598,7 @@ pub fn nodes(input: NomSpan) -> IResult<NomSpan, TokenNode> {

Ok((
input,
TokenTreeBuilder::tagged_token_list(tokens.item, tokens.tag),
TokenTreeBuilder::spanned_token_list(tokens.item, tokens.span),
))
}

Expand Down Expand Up @@ -800,30 +800,30 @@ mod tests {
">" -> b::token_list(vec![b::op(">")])
}

// assert_leaf! {
// parsers [ operator ]
// ">=" -> 0..2 { Operator(Operator::GreaterThanOrEqual) }
// }

// assert_leaf! {
// parsers [ operator ]
// "<" -> 0..1 { Operator(Operator::LessThan) }
// }

// assert_leaf! {
// parsers [ operator ]
// "<=" -> 0..2 { Operator(Operator::LessThanOrEqual) }
// }

// assert_leaf! {
// parsers [ operator ]
// "==" -> 0..2 { Operator(Operator::Equal) }
// }

// assert_leaf! {
// parsers [ operator ]
// "!=" -> 0..2 { Operator(Operator::NotEqual) }
// }
equal_tokens! {
<nodes>
">=" -> b::token_list(vec![b::op(">=")])
}

equal_tokens! {
<nodes>
"<" -> b::token_list(vec![b::op("<")])
}

equal_tokens! {
<nodes>
"<=" -> b::token_list(vec![b::op("<=")])
}

equal_tokens! {
<nodes>
"==" -> b::token_list(vec![b::op("==")])
}

equal_tokens! {
<nodes>
"!=" -> b::token_list(vec![b::op("!=")])
}
}

#[test]
Expand All @@ -848,12 +848,14 @@ mod tests {
}

#[test]
fn test_simple_path() {
fn test_unit_sizes() {
equal_tokens! {
<nodes>
"450MB" -> b::token_list(vec![b::bare("450MB")])
}

}
#[test]
fn test_simple_path() {
equal_tokens! {
<nodes>
"chrome.exe" -> b::token_list(vec![b::bare("chrome"), b::op(Operator::Dot), b::bare("exe")])
Expand All @@ -877,23 +879,23 @@ mod tests {

#[test]
fn test_flag() {
// assert_leaf! {
// parsers [ flag ]
// "--hello" -> 0..7 { Flag(Tagged::from_item(FlagKind::Longhand, span(2, 7))) }
// }

// assert_leaf! {
// parsers [ flag ]
// "--hello-world" -> 0..13 { Flag(Tagged::from_item(FlagKind::Longhand, span(2, 13))) }
// }
equal_tokens! {
<nodes>
"--amigos" -> b::token_list(vec![b::flag("arepas")])
}

equal_tokens! {
<nodes>
"--all-amigos" -> b::token_list(vec![b::flag("all-amigos")])
}
}

#[test]
fn test_shorthand() {
// assert_leaf! {
// parsers [ shorthand ]
// "-alt" -> 0..4 { Flag(Tagged::from_item(FlagKind::Shorthand, span(1, 4))) }
// }
fn test_shorthand_flag() {
equal_tokens! {
<nodes>
"-katz" -> b::token_list(vec![b::shorthand("katz")])
}
}

#[test]
Expand Down Expand Up @@ -939,13 +941,13 @@ mod tests {

equal_tokens! {
<nodes>
"( abc def )" -> b::token_list(vec![b::parens(vec![b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" ")])])
"( abc def )" -> b::token_list(vec![b::parens(vec![b::ws(" "), b::bare("abc"), b::sp(), b::bare("def"), b::sp()])])
}

equal_tokens! {
<nodes>
"( abc def 123 456GB )" -> b::token_list(vec![b::parens(vec![
b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" "), b::int(123), b::ws(" "), b::bare("456GB"), b::ws(" ")
b::ws(" "), b::bare("abc"), b::sp(), b::bare("def"), b::sp(), b::int(123), b::sp(), b::bare("456GB"), b::sp()
])])
}
}
Expand All @@ -964,13 +966,13 @@ mod tests {

equal_tokens! {
<nodes>
"[ abc def ]" -> b::token_list(vec![b::square(vec![b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" ")])])
"[ abc def ]" -> b::token_list(vec![b::square(vec![b::ws(" "), b::bare("abc"), b::sp(), b::bare("def"), b::sp()])])
}

equal_tokens! {
<nodes>
"[ abc def 123 456GB ]" -> b::token_list(vec![b::square(vec![
b::ws(" "), b::bare("abc"), b::ws(" "), b::bare("def"), b::ws(" "), b::int(123), b::ws(" "), b::bare("456GB"), b::ws(" ")
b::ws(" "), b::bare("abc"), b::sp(), b::bare("def"), b::sp(), b::int(123), b::sp(), b::bare("456GB"), b::sp()
])])
}
}
Expand All @@ -984,6 +986,11 @@ mod tests {
"$it.print" -> b::token_list(vec![b::var("it"), b::op("."), b::bare("print")])
}

equal_tokens! {
<nodes>
"$it.0" -> b::token_list(vec![b::var("it"), b::op("."), b::int(0)])
}

equal_tokens! {
<nodes>
"$head.part1.part2" -> b::token_list(vec![b::var("head"), b::op("."), b::bare("part1"), b::op("."), b::bare("part2")])
Expand Down Expand Up @@ -1024,6 +1031,19 @@ mod tests {
b::op("."), b::string("world")]
)
}

equal_tokens! {
<nodes>
r#"$it."are PAS".0"# -> b::token_list(
vec![
b::var("it"),
b::op("."),
b::string("are PAS"),
b::op("."),
b::int(0),
]
)
}
}

#[test]
Expand Down Expand Up @@ -1062,6 +1082,19 @@ mod tests {
<nodes>
"config --set tabs 2" -> b::token_list(vec![b::bare("config"), b::sp(), b::flag("set"), b::sp(), b::bare("tabs"), b::sp(), b::int(2)])
}

equal_tokens! {
<nodes>
"inc --patch package.version" -> b::token_list(
vec![
b::bare("inc"),
b::sp(),
b::flag("patch"),
b::sp(),
b::bare("package"), b::op("."), b::bare("version")
]
)
}
}

#[test]
Expand Down Expand Up @@ -1114,41 +1147,39 @@ mod tests {
fn test_patterns() {
equal_tokens! {
<pipeline>
"cp ../formats/*" -> b::pipeline(vec![vec![b::bare("cp"), b::ws(" "), b::op("."), b::op("."), b::pattern("/formats/*")]])
"cp ../formats/*" -> b::pipeline(vec![vec![b::bare("cp"), b::sp(), b::op("."), b::op("."), b::pattern("/formats/*")]])
}

equal_tokens! {
<pipeline>
"cp * /dev/null" -> b::pipeline(vec![vec![b::bare("cp"), b::ws(" "), b::pattern("*"), b::ws(" "), b::bare("/dev/null")]])
"cp * /dev/null" -> b::pipeline(vec![vec![b::bare("cp"), b::sp(), b::pattern("*"), b::sp(), b::bare("/dev/null")]])
}
}

// #[test]
// fn test_pseudo_paths() {
// let _ = pretty_env_logger::try_init();
#[test]
fn test_pseudo_paths() {
let _ = pretty_env_logger::try_init();

// equal_tokens!(
// r#"sys | where cpu."max ghz" > 1"# ->
// b::pipeline(vec![
// (None, b::call(b::bare("sys"), vec![]), Some(" ")),
// (
// Some(" "),
// b::call(
// b::bare("where"),
// vec![
// b::sp(),
// b::path(b::bare("cpu"), vec![b::string("max ghz")]),
// b::sp(),
// b::op(">"),
// b::sp(),
// b::int(1)
// ]
// ),
// None
// )
// ])
// );
// }
equal_tokens!(
<pipeline>
r#"sys | where cpu."max ghz" > 1"# -> b::pipeline(vec![
vec![
b::bare("sys"), b::sp()
],
vec![
b::sp(),
b::bare("where"),
b::sp(),
b::bare("cpu"),
b::op("."),
b::string("max ghz"),
b::sp(),
b::op(">"),
b::sp(),
b::int(1)
]])
);
}

// #[test]
// fn test_smoke_pipeline() {
Expand Down
2 changes: 1 addition & 1 deletion src/parser/parse/token_tree.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ impl TokenNode {
item: RawToken::Bare,
span,
}) => *span,
other => panic!("Expected var, found {:?}", other),
other => panic!("Expected bare, found {:?}", other),
}
}
}
Expand Down
14 changes: 7 additions & 7 deletions src/parser/parse/token_tree_builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ pub struct TokenTreeBuilder {
}

pub type CurriedToken = Box<dyn FnOnce(&mut TokenTreeBuilder) -> TokenNode + 'static>;
pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Tagged<CallNode> + 'static>;
pub type CurriedCall = Box<dyn FnOnce(&mut TokenTreeBuilder) -> Spanned<CallNode> + 'static>;

impl TokenTreeBuilder {
pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> (TokenNode, String) {
Expand Down Expand Up @@ -89,12 +89,12 @@ impl TokenTreeBuilder {
let tokens = input.into_iter().map(|i| i(b)).collect();
let end = b.pos;

TokenTreeBuilder::tagged_token_list(tokens, (start, end, None))
TokenTreeBuilder::spanned_token_list(tokens, Span::new(start, end))
})
}

pub fn tagged_token_list(input: Vec<TokenNode>, tag: impl Into<Tag>) -> TokenNode {
TokenNode::Nodes(input.spanned(tag.into().span))
pub fn spanned_token_list(input: Vec<TokenNode>, span: impl Into<Span>) -> TokenNode {
TokenNode::Nodes(input.spanned(span.into()))
}

pub fn op(input: impl Into<Operator>) -> CurriedToken {
Expand Down Expand Up @@ -287,11 +287,11 @@ impl TokenTreeBuilder {

let end = b.pos;

TokenTreeBuilder::tagged_call(nodes, (start, end, None))
TokenTreeBuilder::spanned_call(nodes, Span::new(start, end))
})
}

pub fn tagged_call(input: Vec<TokenNode>, tag: impl Into<Tag>) -> Tagged<CallNode> {
pub fn spanned_call(input: Vec<TokenNode>, span: impl Into<Span>) -> Spanned<CallNode> {
if input.len() == 0 {
panic!("BUG: spanned call (TODO)")
}
Expand All @@ -301,7 +301,7 @@ impl TokenTreeBuilder {
let head = input.next().unwrap();
let tail = input.collect();

CallNode::new(Box::new(head), tail).tagged(tag.into())
CallNode::new(Box::new(head), tail).spanned(span.into())
}

fn consume_delimiter(
Expand Down
8 changes: 4 additions & 4 deletions src/parser/parse/tokens.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,14 @@ pub enum RawToken {
impl RawToken {
pub fn type_name(&self) -> &'static str {
match self {
RawToken::Number(_) => "Number",
RawToken::Number(_) => "number",
RawToken::Operator(..) => "operator",
RawToken::String(_) => "String",
RawToken::String(_) => "string",
RawToken::Variable(_) => "variable",
RawToken::ExternalCommand(_) => "external command",
RawToken::ExternalWord => "external word",
RawToken::GlobPattern => "glob pattern",
RawToken::Bare => "String",
RawToken::Bare => "string",
}
}
}
Expand Down Expand Up @@ -72,7 +72,7 @@ impl Token {

pub fn extract_number(&self) -> Option<Spanned<RawNumber>> {
match self.item {
RawToken::Number(number) => Some((number).spanned(self.span)),
RawToken::Number(number) => Some(number.spanned(self.span)),
_ => None,
}
}
Expand Down

0 comments on commit 1b3a094

Please sign in to comment.