From 4f3a5f03005f77c30696a91ceae5b493736af3f8 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Wed, 12 Jun 2019 22:33:38 -0700 Subject: [PATCH] Paths --- Cargo.lock | 10 +++ Cargo.toml | 1 + src/parser/parse2/parser.rs | 131 ++++++++++++++++++++++++++++++-- src/parser/parse2/token_tree.rs | 7 ++ src/parser/parse2/tokens.rs | 1 + 5 files changed, 145 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d72a03bb9..a4de3e0f90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1558,6 +1558,14 @@ dependencies = [ "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "nom-trace" +version = "0.1.0" +source = "git+https://github.com/pythondude325/nom-trace.git#02f45edd3aa4ca9c82addb9c9a22e81044536e99" +dependencies = [ + "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "nom_locate" version = "0.3.1" @@ -1605,6 +1613,7 @@ dependencies = [ "logos 0.10.0-rc2 (registry+https://github.com/rust-lang/crates.io-index)", "logos-derive 0.10.0-rc2 (registry+https://github.com/rust-lang/crates.io-index)", "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "nom-trace 0.1.0 (git+https://github.com/pythondude325/nom-trace.git)", "nom_locate 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "ordered-float 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "pancurses 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3367,6 +3376,7 @@ dependencies = [ "checksum nix 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c722bee1037d430d0f8e687bbdbf222f27cc6e4e68d5caf630857bb2b6dbdce" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +"checksum nom-trace 0.1.0 (git+https://github.com/pythondude325/nom-trace.git)" = "" "checksum nom_locate 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6a47c112b3861d81f7fbf73892b9271af933af32bd5dee6889aa3c3fa9caed7e" "checksum num 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cf4825417e1e1406b3782a8ce92f4d53f26ec055e3622e1881ca8e9f5f9e08db" "checksum num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "107b9be86cd2481930688277b675b0114578227f034674726605b8a482d8baf8" diff --git a/Cargo.toml b/Cargo.toml index 95d677566a..ce9b612010 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ nom_locate = "0.3.1" derive_more = "0.15.0" enum-utils = "0.1.0" unicode-xid = "0.1.0" +nom-trace = { version = "0.1.0", git = "https://github.com/pythondude325/nom-trace.git" } [dependencies.pancurses] version = "0.16" diff --git a/src/parser/parse2/parser.rs b/src/parser/parse2/parser.rs index 74bbc66394..dc4c7909c0 100644 --- a/src/parser/parse2/parser.rs +++ b/src/parser/parse2/parser.rs @@ -2,6 +2,7 @@ use crate::parser::parse2::{flag::*, operator::*, span::*, token_tree::*, tokens::*, unit::*}; use nom; +use nom::dbg; use nom::types::CompleteStr; use nom::*; use nom_locate::{position, LocatedSpan}; @@ -102,13 +103,13 @@ named!(pub var( NomSpan ) -> Token, ) ); -named!(pub identifier( NomSpan ) -> Spanned<()>, +named!(pub identifier( NomSpan ) -> Token, do_parse!( l: position!() >> take_while1!(is_id_start) >> take_while!(is_id_continue) >> r: position!() - >> (Spanned::from_nom((), l, r)) + >> (Spanned::from_nom(RawToken::Identifier, l, r)) ) ); @@ -159,7 +160,7 @@ named!(pub size( NomSpan ) -> Token, // ) named!(pub leaf( NomSpan ) -> Token, - alt!(size | integer | string | operator | flag | shorthand | bare) + alt!(size | integer | string | operator | flag | shorthand | var | bare) ); named!(pub leaf_node( NomSpan ) -> TokenNode, @@ -182,10 +183,25 @@ named!(pub delimited_paren( NomSpan ) -> TokenNode, ) ); -named!(pub node( NomSpan ) -> TokenNode, +named!(pub path( NomSpan ) -> TokenNode, + do_parse!( + l: position!() + >> head: node1 + >> tag!(".") + >> tail: separated_list!(tag!("."), alt!(identifier | string)) + >> r: position!() + >> (TokenNode::Path(Spanned::from_nom(PathNode::new(Box::new(head), tail), l, r))) + ) +); + +named!(pub node1( NomSpan ) -> TokenNode, alt!(leaf_node | delimited_paren) ); +named!(pub node( NomSpan ) -> TokenNode, + alt!(path | leaf_node | delimited_paren) +); + fn int(frag: &str, neg: Option) -> i64 { let int = FromStr::from_str(frag).unwrap(); @@ -237,6 +253,7 @@ fn is_id_continue(c: char) -> bool { #[cfg(test)] mod tests { use super::*; + use nom_trace::{print_trace, reset_trace}; use pretty_assertions::assert_eq; macro_rules! assert_leaf { @@ -390,6 +407,7 @@ mod tests { } } + #[test] fn test_variable() { assert_leaf! { parsers [ var ] @@ -453,8 +471,101 @@ mod tests { ); } + #[test] + fn test_path() { + assert_eq!( + apply(node, "$it.print"), + path( + TokenNode::Token(token(RawToken::Variable(Span::from((1, 3))), 0, 3)), + vec![token(RawToken::Identifier, 4, 9)], + 0, + 9 + ) + ); + + assert_eq!( + apply(node, "$head.part1.part2"), + path( + TokenNode::Token(token(RawToken::Variable(Span::from((1, 5))), 0, 5)), + vec![ + token(RawToken::Identifier, 6, 11), + token(RawToken::Identifier, 12, 17) + ], + 0, + 17 + ) + ); + + assert_eq!( + apply(node, "( hello ).world"), + path( + delimited( + Delimiter::Paren, + vec![TokenNode::Token(token(RawToken::Bare, 2, 7))], + 0, + 9 + ), + vec![token(RawToken::Identifier, 10, 15)], + 0, + 15 + ) + ); + + assert_eq!( + apply(node, "( hello ).\"world\""), + path( + delimited( + Delimiter::Paren, + vec![TokenNode::Token(token(RawToken::Bare, 2, 7))], + 0, + 9 + ), + vec![token(RawToken::String(Span::from((11, 16))), 10, 17)], + 0, + 17 + ) + ); + } + + #[test] + fn test_nested_path() { + assert_eq!( + apply(node, "( $it.is.\"great news\".right yep $yep ).\"world\""), + path( + delimited( + Delimiter::Paren, + vec![ + path( + TokenNode::Token(token(RawToken::Variable(Span::from((3, 5))), 2, 5)), + vec![ + token(RawToken::Identifier, 6, 8), + token(RawToken::String(Span::from((10, 20))), 9, 21), + token(RawToken::Identifier, 22, 27) + ], + 2, + 27 + ), + leaf_token(RawToken::Bare, 28, 31), + leaf_token(RawToken::Variable(Span::from((33, 36))), 32, 36) + ], + 0, + 38 + ), + vec![token(RawToken::String(Span::from((40, 45))), 39, 46)], + 0, + 46 + ) + ); + } + fn apply(f: impl Fn(NomSpan) -> Result<(NomSpan, T), nom::Err>, string: &str) -> T { - f(NomSpan::new(CompleteStr(string))).unwrap().1 + match f(NomSpan::new(CompleteStr(string))) { + Ok(v) => v.1, + Err(other) => { + println!("{:?}", other); + panic!("No dice"); + } + } } fn span(left: usize, right: usize) -> Span { @@ -472,6 +583,16 @@ mod tests { TokenNode::Delimited(spanned) } + fn path(head: TokenNode, tail: Vec, left: usize, right: usize) -> TokenNode { + let node = PathNode::new(Box::new(head), tail); + let spanned = Spanned::from_item(node, (left, right)); + TokenNode::Path(spanned) + } + + fn leaf_token(token: RawToken, left: usize, right: usize) -> TokenNode { + TokenNode::Token(Spanned::from_item(token, (left, right))) + } + fn token(token: RawToken, left: usize, right: usize) -> Token { Spanned::from_item(token, (left, right)) } diff --git a/src/parser/parse2/token_tree.rs b/src/parser/parse2/token_tree.rs index ecb4094283..f79dd20554 100644 --- a/src/parser/parse2/token_tree.rs +++ b/src/parser/parse2/token_tree.rs @@ -6,6 +6,7 @@ use enum_utils::FromStr; pub enum TokenNode { Token(Token), Delimited(Spanned), + Path(Spanned), } #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] @@ -20,3 +21,9 @@ pub enum Delimiter { Brace, Square, } + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, new)] +pub struct PathNode { + head: Box, + tail: Vec, +} diff --git a/src/parser/parse2/tokens.rs b/src/parser/parse2/tokens.rs index eece2cd072..d1977498ce 100644 --- a/src/parser/parse2/tokens.rs +++ b/src/parser/parse2/tokens.rs @@ -10,6 +10,7 @@ pub enum RawToken { Operator(Operator), String(Span), Variable(Span), + Identifier, Bare, Flag(Flag, Span), }