From 57af9b5040c8b8f682a2ebdae4f6cfef48df9ad4 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Wed, 4 Dec 2019 13:14:52 -0800 Subject: [PATCH] Add Range and start Signature support This commit contains two improvements: - Support for a Range syntax (and a corresponding Range value) - Work towards a signature syntax Implementing the Range syntax resulted in cleaning up how operators in the core syntax works. There are now two kinds of infix operators - tight operators (`.` and `..`) - loose operators Tight operators may not be interspersed (`$it.left..$it.right` is a syntax error). Loose operators require whitespace on both sides of the operator, and can be arbitrarily interspersed. Precedence is left to right in the core syntax. Note that delimited syntax (like `( ... )` or `[ ... ]`) is a single token node in the core syntax. A single token node can be parsed from beginning to end in a context-free manner. The rule for `.` is `.`. The rule for `..` is `..`. Loose operators all have the same syntactic rule: ``. The second aspect of this pull request is the beginning of support for a signature syntax. Before implementing signatures, a necessary prerequisite is for the core syntax to support multi-line programs. That work establishes a few things: - `;` and newlines are handled in the core grammar, and both count as "separators" - line comments begin with `#` and continue until the end of the line In this commit, multi-token productions in the core grammar can use separators interchangably with spaces. However, I think we will ultimately want a different rule preventing separators from occurring before an infix operator, so that the end of a line is always unambiguous. This would avoid gratuitous differences between modules and repl usage. We already effectively have this rule, because otherwise `x | y` would be a single pipeline, but of course that wouldn't work. --- .azure/azure-pipelines.yml | 4 +- .gitpod.yml | 2 +- Cargo.lock | 29 + Cargo.toml | 9 +- README.md | 2 +- TODO.md | 4 + crates/nu-errors/src/lib.rs | 17 + crates/nu-macros/Cargo.toml | 10 + crates/nu-macros/src/lib.rs | 25 + crates/nu-parser/Cargo.toml | 3 +- crates/nu-parser/src/hir.rs | 43 +- crates/nu-parser/src/hir/baseline_parse.rs | 2 +- .../nu-parser/src/hir/baseline_parse/tests.rs | 22 +- crates/nu-parser/src/hir/binary.rs | 4 +- .../src/hir/expand_external_tokens.rs | 31 +- crates/nu-parser/src/hir/range.rs | 33 + crates/nu-parser/src/hir/signature.rs | 481 +++++++++++++ crates/nu-parser/src/hir/syntax_shape.rs | 21 +- .../nu-parser/src/hir/syntax_shape/block.rs | 1 + .../src/hir/syntax_shape/expression.rs | 1 + .../src/hir/syntax_shape/expression/atom.rs | 213 ++++-- .../src/hir/syntax_shape/expression/number.rs | 9 +- .../hir/syntax_shape/expression/pattern.rs | 7 +- .../src/hir/syntax_shape/expression/range.rs | 154 +++++ .../src/hir/syntax_shape/expression/string.rs | 3 +- .../syntax_shape/expression/variable_path.rs | 34 +- .../src/hir/syntax_shape/flat_shape.rs | 18 +- crates/nu-parser/src/hir/tokens_iterator.rs | 9 +- crates/nu-parser/src/lib.rs | 13 +- crates/nu-parser/src/parse.rs | 1 + crates/nu-parser/src/parse/comment.rs | 42 ++ crates/nu-parser/src/parse/files.rs | 64 +- crates/nu-parser/src/parse/operator.rs | 91 ++- crates/nu-parser/src/parse/parser.rs | 638 ++++++++++++++---- crates/nu-parser/src/parse/token_tree.rs | 59 +- .../nu-parser/src/parse/token_tree_builder.rs | 68 +- crates/nu-parser/src/parse/tokens.rs | 16 +- crates/nu-protocol/src/lib.rs | 3 + crates/nu-protocol/src/signature.rs | 19 +- crates/nu-protocol/src/syntax_shape.rs | 2 + crates/nu-protocol/src/type_shape.rs | 382 +++++++++++ crates/nu-protocol/src/value.rs | 28 +- crates/nu-protocol/src/value/debug.rs | 16 + crates/nu-protocol/src/value/primitive.rs | 35 +- crates/nu-protocol/src/value/range.rs | 32 + crates/nu-source/src/meta.rs | 27 +- crates/nu_plugin_post/src/main.rs | 4 +- src/commands/cd.rs | 17 +- src/commands/command.rs | 18 +- src/commands/range.rs | 57 +- src/commands/to_bson.rs | 2 +- src/commands/to_json.rs | 4 +- src/commands/to_sqlite.rs | 7 +- src/commands/to_toml.rs | 1 + src/commands/to_yaml.rs | 4 +- src/data/base.rs | 4 +- src/data/base/shape.rs | 259 +------ src/data/value.rs | 29 +- src/deserializer.rs | 32 +- src/evaluate/evaluator.rs | 26 +- src/evaluate/operator.rs | 23 +- src/shell/helper.rs | 6 +- tests/command_cd_tests.rs | 16 +- tests/commands_test.rs | 24 - 64 files changed, 2522 insertions(+), 738 deletions(-) create mode 100644 crates/nu-macros/Cargo.toml create mode 100644 crates/nu-macros/src/lib.rs create mode 100644 crates/nu-parser/src/hir/range.rs create mode 100644 crates/nu-parser/src/hir/signature.rs create mode 100644 crates/nu-parser/src/hir/syntax_shape/expression/range.rs create mode 100644 crates/nu-parser/src/parse/comment.rs create mode 100644 crates/nu-protocol/src/type_shape.rs create mode 100644 crates/nu-protocol/src/value/range.rs diff --git a/.azure/azure-pipelines.yml b/.azure/azure-pipelines.yml index 70ff5e4af0..a963afc2ec 100644 --- a/.azure/azure-pipelines.yml +++ b/.azure/azure-pipelines.yml @@ -42,10 +42,10 @@ steps: echo "##vso[task.prependpath]$HOME/.cargo/bin" rustup component add rustfmt --toolchain "stable" displayName: Install Rust - - bash: RUSTFLAGS="-D warnings" cargo test --all --all-features + - bash: RUSTFLAGS="-D warnings" cargo test --all --features=user-visible condition: eq(variables['style'], 'unflagged') displayName: Run tests - - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all --all-features + - bash: NUSHELL_ENABLE_ALL_FLAGS=1 RUSTFLAGS="-D warnings" cargo test --all --features=user-visible condition: eq(variables['style'], 'canary') displayName: Run tests - bash: cargo fmt --all -- --check diff --git a/.gitpod.yml b/.gitpod.yml index 8bbe78cf88..28a826a94f 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,7 +1,7 @@ image: file: .gitpod.Dockerfile tasks: - - init: cargo install nu --all-features + - init: cargo install nu --features=user-visible command: nu github: prebuilds: diff --git a/Cargo.lock b/Cargo.lock index 1504f4612e..772bb51497 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -814,6 +814,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enumflags2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33121c8782ba948ba332dab29311b026a8716dc65a1599e5b88f392d38496af8" +dependencies = [ + "enumflags2_derive", +] + +[[package]] +name = "enumflags2_derive" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf634c5213044b8d54a46dd282cf5dd1f86bb5cb53e92c409cb4680a7fb9894" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "env_logger" version = "0.6.2" @@ -1896,6 +1916,7 @@ dependencies = [ "nom_locate", "nu-build", "nu-errors", + "nu-macros", "nu-parser", "nu-protocol", "nu-source", @@ -1983,6 +2004,13 @@ dependencies = [ "toml 0.5.5", ] +[[package]] +name = "nu-macros" +version = "0.1.0" +dependencies = [ + "nu-protocol", +] + [[package]] name = "nu-parser" version = "0.1.0" @@ -1991,6 +2019,7 @@ dependencies = [ "bigdecimal", "cfg-if", "derive-new", + "enumflags2", "getset", "indexmap", "itertools 0.8.2", diff --git a/Cargo.toml b/Cargo.toml index d49f731735..bdeae804be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ documentation = "https://book.nushell.sh" [workspace] members = [ + "crates/nu-macros", "crates/nu-errors", "crates/nu-source", "crates/nu_plugin_average", @@ -54,6 +55,8 @@ nu_plugin_sum = {version = "0.1.0", path = "./crates/nu_plugin_sum", optional=tr nu_plugin_sys = {version = "0.1.0", path = "./crates/nu_plugin_sys", optional=true} nu_plugin_textview = {version = "0.1.0", path = "./crates/nu_plugin_textview", optional=true} nu_plugin_tree = {version = "0.1.0", path = "./crates/nu_plugin_tree", optional=true} +nu-macros = { version = "0.1.0", path = "./crates/nu-macros" } + query_interface = "0.3.5" typetag = "0.1.4" @@ -133,19 +136,21 @@ semver = {version = "0.9.0", optional = true} [features] default = ["sys", "ps", "textview", "inc", "str"] +user-visible = ["sys", "ps", "starship-prompt", "textview", "binaryview", "match", "tree", "average", "sum"] + sys = ["heim", "battery"] ps = ["heim", "futures-timer"] textview = ["crossterm", "syntect", "onig_sys", "url"] -inc = ["semver"] str = [] +inc = ["semver"] starship-prompt = ["starship"] binaryview = ["nu_plugin_binaryview"] match = ["nu_plugin_match"] tree = ["nu_plugin_tree"] average = ["nu_plugin_average"] sum = ["nu_plugin_sum"] -#trace = ["nu-parser/trace"] +trace = ["nu-parser/trace"] [dependencies.rusqlite] version = "0.20.0" diff --git a/README.md b/README.md index c1ae7a02f8..44264b843b 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ cargo install nu You can also install Nu with all the bells and whistles (be sure to have installed the [dependencies](https://book.nushell.sh/en/installation#dependencies) for your platform): ``` -cargo install nu --all-features +cargo install nu --features=user-visible ``` ## Docker diff --git a/TODO.md b/TODO.md index cc7c1e5ec5..db001731f2 100644 --- a/TODO.md +++ b/TODO.md @@ -46,3 +46,7 @@ Unify dictionary building, probably around a macro sys plugin in own crate textview in own crate + +Combine atomic and atomic_parse in parser + +at_end_possible_ws needs to be comment and separator sensitive diff --git a/crates/nu-errors/src/lib.rs b/crates/nu-errors/src/lib.rs index 1abeae9db4..231a93ba95 100644 --- a/crates/nu-errors/src/lib.rs +++ b/crates/nu-errors/src/lib.rs @@ -16,6 +16,9 @@ use std::ops::Range; pub enum ParseErrorReason { /// The parser encountered an EOF rather than what it was expecting Eof { expected: &'static str, span: Span }, + /// The parser expected to see the end of a token stream (possibly the token + /// stream from inside a delimited token node), but found something else. + ExtraTokens { actual: Spanned }, /// The parser encountered something other than what it was expecting Mismatch { expected: &'static str, @@ -43,6 +46,17 @@ impl ParseError { } } + /// Construct a [ParseErrorReason::ExtraTokens](ParseErrorReason::ExtraTokens) + pub fn extra_tokens(actual: Spanned>) -> ParseError { + let Spanned { span, item } = actual; + + ParseError { + reason: ParseErrorReason::ExtraTokens { + actual: item.into().spanned(span), + }, + } + } + /// Construct a [ParseErrorReason::Mismatch](ParseErrorReason::Mismatch) pub fn mismatch(expected: &'static str, actual: Spanned>) -> ParseError { let Spanned { span, item } = actual; @@ -71,6 +85,9 @@ impl From for ShellError { fn from(error: ParseError) -> ShellError { match error.reason { ParseErrorReason::Eof { expected, span } => ShellError::unexpected_eof(expected, span), + ParseErrorReason::ExtraTokens { actual } => { + ShellError::type_error("nothing", actual.clone()) + } ParseErrorReason::Mismatch { actual, expected } => { ShellError::type_error(expected, actual.clone()) } diff --git a/crates/nu-macros/Cargo.toml b/crates/nu-macros/Cargo.toml new file mode 100644 index 0000000000..cddf28a1c8 --- /dev/null +++ b/crates/nu-macros/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "nu-macros" +version = "0.1.0" +authors = ["Yehuda Katz "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +nu-protocol = { path = "../nu-protocol", version = "0.1.0" } \ No newline at end of file diff --git a/crates/nu-macros/src/lib.rs b/crates/nu-macros/src/lib.rs new file mode 100644 index 0000000000..0c323f9aa1 --- /dev/null +++ b/crates/nu-macros/src/lib.rs @@ -0,0 +1,25 @@ +#[macro_export] +macro_rules! signature { + (def $name:tt { + $usage:tt + $( + $positional_name:tt $positional_ty:tt - $positional_desc:tt + )* + }) => {{ + let signature = Signature::new(stringify!($name)).desc($usage); + $( + $crate::positional! { signature, $positional_name $positional_ty - $positional_desc } + )* + signature + }}; +} + +#[macro_export] +macro_rules! positional { + ($ident:tt, $name:tt (optional $shape:tt) - $desc:tt) => { + let $ident = $ident.required(stringify!($name), SyntaxShape::$shape, $desc); + }; + ($ident:tt, $name:tt ($shape:tt)- $desc:tt) => { + let $ident = $ident.optional(stringify!($name), SyntaxShape::$shape, $desc); + }; +} diff --git a/crates/nu-parser/Cargo.toml b/crates/nu-parser/Cargo.toml index 804d3183fd..6a6038d517 100644 --- a/crates/nu-parser/Cargo.toml +++ b/crates/nu-parser/Cargo.toml @@ -32,6 +32,7 @@ ansi_term = "0.12.1" ptree = {version = "0.2" } language-reporting = "0.4.0" unicode-xid = "0.2.0" +enumflags2 = "0.6.2" [dev-dependencies] pretty_assertions = "0.6.1" @@ -40,4 +41,4 @@ pretty_assertions = "0.6.1" nu-build = { version = "0.1.0", path = "../nu-build" } [features] -#trace = ["nom-tracable/trace"] +trace = ["nom-tracable/trace"] diff --git a/crates/nu-parser/src/hir.rs b/crates/nu-parser/src/hir.rs index 42e01e2d6a..6c79d77ce6 100644 --- a/crates/nu-parser/src/hir.rs +++ b/crates/nu-parser/src/hir.rs @@ -4,11 +4,13 @@ pub(crate) mod expand_external_tokens; pub(crate) mod external_command; pub(crate) mod named; pub(crate) mod path; +pub(crate) mod range; +pub(crate) mod signature; pub mod syntax_shape; pub(crate) mod tokens_iterator; use crate::hir::syntax_shape::Member; -use crate::parse::operator::Operator; +use crate::parse::operator::CompareOperator; use crate::parse::parser::Number; use crate::parse::unit::Unit; use derive_new::new; @@ -24,12 +26,40 @@ use crate::parse::tokens::RawNumber; pub(crate) use self::binary::Binary; pub(crate) use self::path::Path; +pub(crate) use self::range::Range; pub(crate) use self::syntax_shape::ExpandContext; pub(crate) use self::tokens_iterator::TokensIterator; pub use self::external_command::ExternalCommand; pub use self::named::{NamedArguments, NamedValue}; +#[derive(Debug, Clone)] +pub struct Signature { + unspanned: nu_protocol::Signature, + span: Span, +} + +impl Signature { + pub fn new(unspanned: nu_protocol::Signature, span: impl Into) -> Signature { + Signature { + unspanned, + span: span.into(), + } + } +} + +impl HasSpan for Signature { + fn span(&self) -> Span { + self.span + } +} + +impl PrettyDebugWithSource for Signature { + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + self.unspanned.pretty_debug(source) + } +} + #[derive(Debug, Clone, Eq, PartialEq, Getters, Serialize, Deserialize, new)] pub struct Call { #[get = "pub(crate)"] @@ -68,6 +98,7 @@ pub enum RawExpression { Synthetic(Synthetic), Variable(Variable), Binary(Box), + Range(Box), Block(Vec), List(Vec), Path(Box), @@ -90,6 +121,7 @@ impl ShellTypeName for RawExpression { RawExpression::Variable(..) => "variable", RawExpression::List(..) => "list", RawExpression::Binary(..) => "binary", + RawExpression::Range(..) => "range", RawExpression::Block(..) => "block", RawExpression::Path(..) => "variable path", RawExpression::Boolean(..) => "boolean", @@ -159,6 +191,7 @@ impl PrettyDebugWithSource for Expression { }, RawExpression::Variable(_) => b::keyword(self.span.slice(source)), RawExpression::Binary(binary) => binary.pretty_debug(source), + RawExpression::Range(range) => range.pretty_debug(source), RawExpression::Block(_) => b::opaque("block"), RawExpression::List(list) => b::delimit( "[", @@ -245,7 +278,7 @@ impl Expression { pub fn infix( left: Expression, - op: Spanned>, + op: Spanned>, right: Expression, ) -> Expression { let new_span = left.span.until(right.span); @@ -254,6 +287,12 @@ impl Expression { .into_expr(new_span) } + pub fn range(left: Expression, op: Span, right: Expression) -> Expression { + let new_span = left.span.until(right.span); + + RawExpression::Range(Box::new(Range::new(left, op, right))).into_expr(new_span) + } + pub fn file_path(path: impl Into, outer: impl Into) -> Expression { RawExpression::FilePath(path.into()).into_expr(outer) } diff --git a/crates/nu-parser/src/hir/baseline_parse.rs b/crates/nu-parser/src/hir/baseline_parse.rs index 87c2771955..192cb697cc 100644 --- a/crates/nu-parser/src/hir/baseline_parse.rs +++ b/crates/nu-parser/src/hir/baseline_parse.rs @@ -1,2 +1,2 @@ #[cfg(test)] -mod tests; +pub mod tests; diff --git a/crates/nu-parser/src/hir/baseline_parse/tests.rs b/crates/nu-parser/src/hir/baseline_parse/tests.rs index 216be656b8..d6d3212563 100644 --- a/crates/nu-parser/src/hir/baseline_parse/tests.rs +++ b/crates/nu-parser/src/hir/baseline_parse/tests.rs @@ -23,7 +23,7 @@ fn test_parse_string() { fn test_parse_path() { parse_tokens( VariablePathShape, - vec![b::var("it"), b::op("."), b::bare("cpu")], + vec![b::var("it"), b::dot(), b::bare("cpu")], |tokens| { let (outer_var, inner_var) = tokens[0].expect_var(); let bare = tokens[2].expect_bare(); @@ -39,9 +39,9 @@ fn test_parse_path() { VariablePathShape, vec![ b::var("cpu"), - b::op("."), + b::dot(), b::bare("amount"), - b::op("."), + b::dot(), b::string("max ghz"), ], |tokens| { @@ -145,7 +145,7 @@ fn parse_tokens( let expr = match expr { Ok(expr) => expr, Err(err) => { - print_err(err.into(), context.source().clone()); + print_err(err.into(), &context.source().clone()); panic!("Parse failed"); } }; @@ -165,12 +165,10 @@ pub fn print_err(err: ShellError, source: &Text) { let mut source = source.to_string(); source.push_str(" "); let files = Files::new(source); - let _ = std::panic::catch_unwind(move || { - let _ = language_reporting::emit( - &mut writer.lock(), - &files, - &diag, - &language_reporting::DefaultConfig, - ); - }); + let _ = language_reporting::emit( + &mut writer.lock(), + &files, + &diag, + &language_reporting::DefaultConfig, + ); } diff --git a/crates/nu-parser/src/hir/binary.rs b/crates/nu-parser/src/hir/binary.rs index 47148fa6d8..cfc1e617ef 100644 --- a/crates/nu-parser/src/hir/binary.rs +++ b/crates/nu-parser/src/hir/binary.rs @@ -1,4 +1,4 @@ -use crate::{hir::Expression, Operator}; +use crate::{hir::Expression, CompareOperator}; use derive_new::new; use getset::Getters; @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; #[get = "pub"] pub struct Binary { left: Expression, - op: Spanned, + op: Spanned, right: Expression, } diff --git a/crates/nu-parser/src/hir/expand_external_tokens.rs b/crates/nu-parser/src/hir/expand_external_tokens.rs index 1ed67d44ce..5196a1ab47 100644 --- a/crates/nu-parser/src/hir/expand_external_tokens.rs +++ b/crates/nu-parser/src/hir/expand_external_tokens.rs @@ -10,6 +10,7 @@ use crate::{ TokensIterator, }; use nu_errors::ParseError; +use nu_protocol::SpannedTypeName; use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebug, Span, Spanned, SpannedItem}; #[derive(Debug, Clone)] @@ -195,11 +196,18 @@ impl ExpandExpression for ExternalHeadShape { UnspannedAtomicToken::Whitespace { .. } => { unreachable!("ExpansionRule doesn't allow Whitespace") } + UnspannedAtomicToken::Separator { .. } => { + unreachable!("ExpansionRule doesn't allow Separator") + } + UnspannedAtomicToken::Comment { .. } => { + unreachable!("ExpansionRule doesn't allow Comment") + } UnspannedAtomicToken::ShorthandFlag { .. } - | UnspannedAtomicToken::SquareDelimited { .. } => { + | UnspannedAtomicToken::SquareDelimited { .. } + | UnspannedAtomicToken::RoundDelimited { .. } => { return Err(ParseError::mismatch( "external command name", - "pipeline".spanned(atom.span), + atom.spanned_type_name(), )) } UnspannedAtomicToken::ExternalCommand { command } => { @@ -215,7 +223,10 @@ impl ExpandExpression for ExternalHeadShape { | UnspannedAtomicToken::GlobPattern { .. } | UnspannedAtomicToken::Word { .. } | UnspannedAtomicToken::Dot { .. } - | UnspannedAtomicToken::Operator { .. } => Expression::external_command(span, span), + | UnspannedAtomicToken::DotDot { .. } + | UnspannedAtomicToken::CompareOperator { .. } => { + Expression::external_command(span, span) + } }) } } @@ -257,6 +268,12 @@ impl ExpandExpression for ExternalContinuationShape { UnspannedAtomicToken::Whitespace { .. } => { unreachable!("ExpansionRule doesn't allow Whitespace") } + UnspannedAtomicToken::Separator { .. } => { + unreachable!("ExpansionRule doesn't allow Separator") + } + UnspannedAtomicToken::Comment { .. } => { + unreachable!("ExpansionRule doesn't allow Comment") + } UnspannedAtomicToken::String { body } => Expression::string(*body, span), UnspannedAtomicToken::ItVariable { name } => Expression::it_variable(*name, span), UnspannedAtomicToken::Variable { name } => Expression::variable(*name, span), @@ -265,11 +282,13 @@ impl ExpandExpression for ExternalContinuationShape { | UnspannedAtomicToken::Word { .. } | UnspannedAtomicToken::ShorthandFlag { .. } | UnspannedAtomicToken::Dot { .. } - | UnspannedAtomicToken::Operator { .. } => Expression::bare(span), - UnspannedAtomicToken::SquareDelimited { .. } => { + | UnspannedAtomicToken::DotDot { .. } + | UnspannedAtomicToken::CompareOperator { .. } => Expression::bare(span), + UnspannedAtomicToken::SquareDelimited { .. } + | UnspannedAtomicToken::RoundDelimited { .. } => { return Err(ParseError::mismatch( "external argument", - "pipeline".spanned(atom.span), + atom.spanned_type_name(), )) } }) diff --git a/crates/nu-parser/src/hir/range.rs b/crates/nu-parser/src/hir/range.rs new file mode 100644 index 0000000000..075dd3bd60 --- /dev/null +++ b/crates/nu-parser/src/hir/range.rs @@ -0,0 +1,33 @@ +use crate::hir::Expression; + +use derive_new::new; +use getset::Getters; +use nu_source::{b, DebugDocBuilder, PrettyDebugWithSource, Span}; +use serde::{Deserialize, Serialize}; + +#[derive( + Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Getters, Serialize, Deserialize, new, +)] +pub struct Range { + #[get = "pub"] + left: Expression, + #[get = "pub"] + dotdot: Span, + #[get = "pub"] + right: Expression, +} + +impl PrettyDebugWithSource for Range { + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + b::delimit( + "<", + self.left.pretty_debug(source) + + b::space() + + b::keyword(self.dotdot.slice(source)) + + b::space() + + self.right.pretty_debug(source), + ">", + ) + .group() + } +} diff --git a/crates/nu-parser/src/hir/signature.rs b/crates/nu-parser/src/hir/signature.rs new file mode 100644 index 0000000000..bf6278cb2b --- /dev/null +++ b/crates/nu-parser/src/hir/signature.rs @@ -0,0 +1,481 @@ +use crate::hir; +use crate::hir::syntax_shape::{ + expand_atom, expand_syntax, BareShape, ExpandContext, ExpandSyntax, ExpansionRule, + UnspannedAtomicToken, WhitespaceShape, +}; +use crate::hir::tokens_iterator::TokensIterator; +use crate::parse::comment::Comment; +use derive_new::new; +use nu_errors::ParseError; +use nu_protocol::{RowType, SpannedTypeName, Type}; +use nu_source::{ + b, DebugDocBuilder, HasFallibleSpan, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem, +}; +use std::fmt::Debug; + +// A Signature is a command without implementation. +// +// In Nu, a command is a function combined with macro expansion rules. +// +// def cd +// # Change to a new path. +// optional directory(Path) # the directory to change to +// end + +#[derive(new)] +struct Expander<'a, 'b, 'c, 'd> { + iterator: &'b mut TokensIterator<'a>, + context: &'d ExpandContext<'c>, +} + +impl<'a, 'b, 'c, 'd> Expander<'a, 'b, 'c, 'd> { + fn expand(&mut self, syntax: impl ExpandSyntax) -> Result + where + O: HasFallibleSpan + Clone + std::fmt::Debug + 'static, + { + expand_syntax(&syntax, self.iterator, self.context) + } + + fn optional(&mut self, syntax: impl ExpandSyntax) -> Option + where + O: HasFallibleSpan + Clone + std::fmt::Debug + 'static, + { + match expand_syntax(&syntax, self.iterator, self.context) { + Err(_) => None, + Ok(value) => Some(value), + } + } + + fn pos(&mut self) -> Span { + self.iterator.span_at_cursor() + } + + fn slice_string(&mut self, span: impl Into) -> String { + span.into().slice(self.context.source()).to_string() + } +} + +#[derive(Debug, Copy, Clone)] +struct SignatureShape; + +impl ExpandSyntax for SignatureShape { + type Output = hir::Signature; + + fn name(&self) -> &'static str { + "signature" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + token_nodes.atomic_parse(|token_nodes| { + let mut expander = Expander::new(token_nodes, context); + let start = expander.pos(); + expander.expand(keyword("def"))?; + expander.expand(WhitespaceShape)?; + let name = expander.expand(BareShape)?; + expander.expand(SeparatorShape)?; + let usage = expander.expand(CommentShape)?; + expander.expand(SeparatorShape)?; + let end = expander.pos(); + + Ok(hir::Signature::new( + nu_protocol::Signature::new(&name.word).desc(expander.slice_string(usage.text)), + start.until(end), + )) + }) + } +} + +fn keyword(kw: &'static str) -> KeywordShape { + KeywordShape { keyword: kw } +} + +#[derive(Debug, Copy, Clone)] +struct KeywordShape { + keyword: &'static str, +} + +impl ExpandSyntax for KeywordShape { + type Output = Span; + + fn name(&self) -> &'static str { + "keyword" + } + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let atom = expand_atom(token_nodes, "keyword", context, ExpansionRule::new())?; + + match &atom.unspanned { + UnspannedAtomicToken::Word { text } => { + let word = text.slice(context.source()); + + if word == self.keyword { + return Ok(atom.span); + } + } + _ => {} + } + + Err(ParseError::mismatch(self.keyword, atom.spanned_type_name())) + } +} + +#[derive(Debug, Copy, Clone)] +struct SeparatorShape; + +impl ExpandSyntax for SeparatorShape { + type Output = Span; + + fn name(&self) -> &'static str { + "separator" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let atom = expand_atom(token_nodes, "separator", context, ExpansionRule::new())?; + + match &atom.unspanned { + UnspannedAtomicToken::Separator { text } => Ok(*text), + _ => Err(ParseError::mismatch("separator", atom.spanned_type_name())), + } + } +} + +#[derive(Debug, Copy, Clone)] +struct CommentShape; + +impl ExpandSyntax for CommentShape { + type Output = Comment; + + fn name(&self) -> &'static str { + "comment" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let atom = expand_atom(token_nodes, "comment", context, ExpansionRule::new())?; + + match &atom.unspanned { + UnspannedAtomicToken::Comment { body } => Ok(Comment::line(body, atom.span)), + _ => Err(ParseError::mismatch("separator", atom.spanned_type_name())), + } + } +} + +#[derive(Debug, Copy, Clone, new)] +struct TupleShape { + first: A, + second: B, +} + +#[derive(Debug, Clone, new)] +struct TupleSyntax { + first: A, + second: B, +} + +impl PrettyDebugWithSource for TupleSyntax +where + A: PrettyDebugWithSource, + B: PrettyDebugWithSource, +{ + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + b::typed( + "pair", + self.first.pretty_debug(source) + b::space() + self.second.pretty_debug(source), + ) + } +} + +impl HasFallibleSpan for TupleSyntax +where + A: HasFallibleSpan + Debug + Clone, + B: HasFallibleSpan + Debug + Clone, +{ + fn maybe_span(&self) -> Option { + match (self.first.maybe_span(), self.second.maybe_span()) { + (Some(first), Some(second)) => Some(first.until(second)), + (Some(first), None) => Some(first), + (None, Some(second)) => Some(second), + (None, None) => None, + } + } +} + +impl ExpandSyntax for TupleShape +where + A: ExpandSyntax + Debug + Copy, + B: ExpandSyntax + Debug + Copy, + AOut: HasFallibleSpan + Debug + Clone + 'static, + BOut: HasFallibleSpan + Debug + Clone + 'static, +{ + type Output = TupleSyntax; + + fn name(&self) -> &'static str { + "pair" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + token_nodes.atomic_parse(|token_nodes| { + let first = expand_syntax(&self.first, token_nodes, context)?; + let second = expand_syntax(&self.second, token_nodes, context)?; + + Ok(TupleSyntax { first, second }) + }) + } +} + +#[derive(Debug, Clone)] +pub struct PositionalParam { + optional: Option, + name: Identifier, + ty: Spanned, + desc: Spanned, + span: Span, +} + +impl HasSpan for PositionalParam { + fn span(&self) -> Span { + self.span + } +} + +impl PrettyDebugWithSource for PositionalParam { + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + (match self.optional { + Some(_) => b::description("optional") + b::space(), + None => b::blank(), + }) + self.ty.pretty_debug(source) + } +} + +#[derive(Debug, Copy, Clone)] +pub struct PositionalParamShape; + +impl ExpandSyntax for PositionalParamShape { + type Output = PositionalParam; + + fn name(&self) -> &'static str { + "positional param" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + token_nodes.atomic_parse(|token_nodes| { + let mut expander = Expander::new(token_nodes, context); + + let optional = expander + .optional(TupleShape::new(keyword("optional"), WhitespaceShape)) + .map(|s| s.first); + + let name = expander.expand(IdentifierShape)?; + + expander.optional(WhitespaceShape); + + let _ty = expander.expand(TypeShape)?; + + Ok(PositionalParam { + optional, + name, + ty: Type::Nothing.spanned(Span::unknown()), + desc: format!("").spanned(Span::unknown()), + span: Span::unknown(), + }) + }) + } +} + +#[derive(Debug, Clone)] +struct Identifier { + body: String, + span: Span, +} + +impl HasSpan for Identifier { + fn span(&self) -> Span { + self.span + } +} + +impl PrettyDebugWithSource for Identifier { + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + b::typed("id", b::description(self.span.slice(source))) + } +} + +#[derive(Debug, Copy, Clone)] +struct IdentifierShape; + +impl ExpandSyntax for IdentifierShape { + type Output = Identifier; + fn name(&self) -> &'static str { + "identifier" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let atom = expand_atom(token_nodes, "identifier", context, ExpansionRule::new())?; + + match atom.unspanned { + UnspannedAtomicToken::Word { text } => { + let body = text.slice(context.source()); + if is_id(body) { + return Ok(Identifier { + body: body.to_string(), + span: text, + }); + } + } + _ => {} + } + + Err(ParseError::mismatch("identifier", atom.spanned_type_name())) + } +} + +fn is_id(input: &str) -> bool { + let source = nu_source::nom_input(input); + match crate::parse::parser::ident(source) { + Err(_) => false, + Ok((input, _)) => input.fragment.len() == 0, + } +} + +#[derive(Debug, Clone, new)] +struct TypeSyntax { + ty: Type, + span: Span, +} + +impl HasSpan for TypeSyntax { + fn span(&self) -> Span { + self.span + } +} + +impl PrettyDebugWithSource for TypeSyntax { + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + self.ty.pretty_debug(source) + } +} + +#[derive(Debug, Copy, Clone)] +struct TypeShape; + +impl ExpandSyntax for TypeShape { + type Output = TypeSyntax; + + fn name(&self) -> &'static str { + "type" + } + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let atom = expand_atom(token_nodes, "type", context, ExpansionRule::new())?; + + match atom.unspanned { + UnspannedAtomicToken::Word { text } => { + let word = text.slice(context.source()); + + Ok(TypeSyntax::new( + match word { + "nothing" => Type::Nothing, + "integer" => Type::Int, + "decimal" => Type::Decimal, + "bytesize" => Type::Bytesize, + "string" => Type::String, + "column-path" => Type::ColumnPath, + "pattern" => Type::Pattern, + "boolean" => Type::Boolean, + "date" => Type::Date, + "duration" => Type::Duration, + "filename" => Type::Path, + "binary" => Type::Binary, + "row" => Type::Row(RowType::new()), + "table" => Type::Table(vec![]), + "block" => Type::Block, + _ => return Err(ParseError::mismatch("type", atom.spanned_type_name())), + }, + atom.span, + )) + } + _ => Err(ParseError::mismatch("type", atom.spanned_type_name())), + } + } +} + +#[derive(Debug, Copy, Clone)] +struct TypeAnnotation; + +impl ExpandSyntax for TypeAnnotation { + type Output = TypeSyntax; + + fn name(&self) -> &'static str { + "type annotation" + } + + fn expand_syntax<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + let atom = expand_atom( + token_nodes, + "type annotation", + context, + ExpansionRule::new(), + )?; + + match atom.unspanned { + UnspannedAtomicToken::RoundDelimited { nodes, .. } => { + token_nodes.atomic_parse(|token_nodes| { + token_nodes.child( + (&nodes[..]).spanned(atom.span), + context.source().clone(), + |token_nodes| { + let ty = expand_syntax(&TypeShape, token_nodes, context)?; + + let next = token_nodes.peek_non_ws(); + + match next.node { + None => Ok(ty), + Some(node) => { + Err(ParseError::extra_tokens(node.spanned_type_name())) + } + } + }, + ) + }) + } + + _ => Err(ParseError::mismatch( + "type annotation", + atom.spanned_type_name(), + )), + } + } +} diff --git a/crates/nu-parser/src/hir/syntax_shape.rs b/crates/nu-parser/src/hir/syntax_shape.rs index 04d89e93aa..5659116001 100644 --- a/crates/nu-parser/src/hir/syntax_shape.rs +++ b/crates/nu-parser/src/hir/syntax_shape.rs @@ -7,8 +7,9 @@ use crate::commands::external_command; use crate::hir; use crate::hir::expand_external_tokens::ExternalTokensShape; use crate::hir::syntax_shape::block::AnyBlockShape; +use crate::hir::syntax_shape::expression::range::RangeShape; use crate::hir::tokens_iterator::{Peeked, TokensIterator}; -use crate::parse::operator::Operator; +use crate::parse::operator::EvaluationOperator; use crate::parse::token_tree::TokenNode; use crate::parse::tokens::{Token, UnspannedToken}; use crate::parse_command::{parse_command_tail, CommandTailShape}; @@ -74,6 +75,7 @@ impl FallibleColorSyntax for SyntaxShape { context, shapes, ), + SyntaxShape::Range => color_fallible_syntax(&RangeShape, token_nodes, context, shapes), SyntaxShape::Member => { color_fallible_syntax(&MemberShape, token_nodes, context, shapes) } @@ -114,6 +116,7 @@ impl FallibleColorSyntax for SyntaxShape { match self { SyntaxShape::Any => color_fallible_syntax(&AnyExpressionShape, token_nodes, context), SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context), + SyntaxShape::Range => color_fallible_syntax(&RangeShape, token_nodes, context), SyntaxShape::String => { color_fallible_syntax_with(&StringShape, &FlatShape::String, token_nodes, context) } @@ -134,6 +137,7 @@ impl ExpandExpression for SyntaxShape { match self { SyntaxShape::Any => "shape[any]", SyntaxShape::Int => "shape[integer]", + SyntaxShape::Range => "shape[range]", SyntaxShape::String => "shape[string]", SyntaxShape::Member => "shape[column name]", SyntaxShape::ColumnPath => "shape[column path]", @@ -152,6 +156,7 @@ impl ExpandExpression for SyntaxShape { match self { SyntaxShape::Any => expand_expr(&AnyExpressionShape, token_nodes, context), SyntaxShape::Int => expand_expr(&IntShape, token_nodes, context), + SyntaxShape::Range => expand_expr(&RangeShape, token_nodes, context), SyntaxShape::String => expand_expr(&StringShape, token_nodes, context), SyntaxShape::Member => { let syntax = expand_syntax(&MemberShape, token_nodes, context)?; @@ -183,7 +188,6 @@ pub trait SignatureRegistry { pub struct ExpandContext<'context> { #[get = "pub(crate)"] pub registry: Box, - #[get = "pub(crate)"] pub source: &'context Text, pub homedir: Option, } @@ -192,6 +196,10 @@ impl<'context> ExpandContext<'context> { pub(crate) fn homedir(&self) -> Option<&Path> { self.homedir.as_ref().map(|h| h.as_path()) } + + pub(crate) fn source(&self) -> &'context Text { + self.source + } } pub trait TestSyntax: std::fmt::Debug + Copy { @@ -568,7 +576,7 @@ impl ExpandSyntax for BarePathShape { .. }) | TokenNode::Token(Token { - unspanned: UnspannedToken::Operator(Operator::Dot), + unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::Dot), .. }) => true, @@ -604,7 +612,10 @@ impl FallibleColorSyntax for BareShape { } // otherwise, fail - other => Err(ParseError::mismatch("word", other.spanned_type_name())), + other => Err(ParseError::mismatch( + "word", + other.type_name().spanned(other.span()), + )), }) .map_err(|err| err.into()) } @@ -1600,7 +1611,7 @@ impl FallibleColorSyntax for SpaceShape { other => Err(ShellError::type_error( "whitespace", - other.spanned_type_name(), + other.type_name().spanned(other.span()), )), } } diff --git a/crates/nu-parser/src/hir/syntax_shape/block.rs b/crates/nu-parser/src/hir/syntax_shape/block.rs index 4a8f0aa9ac..c6ad13a0cd 100644 --- a/crates/nu-parser/src/hir/syntax_shape/block.rs +++ b/crates/nu-parser/src/hir/syntax_shape/block.rs @@ -390,6 +390,7 @@ impl FallibleColorSyntax for ShorthandHeadShape { ) -> Result<(), ShellError> { use crate::parse::token_tree::TokenNode; use crate::parse::tokens::{Token, UnspannedToken}; + use nu_protocol::SpannedTypeName; use nu_source::SpannedItem; // A shorthand path must not be at EOF diff --git a/crates/nu-parser/src/hir/syntax_shape/expression.rs b/crates/nu-parser/src/hir/syntax_shape/expression.rs index 4c7dfd3db6..02f68fe1dd 100644 --- a/crates/nu-parser/src/hir/syntax_shape/expression.rs +++ b/crates/nu-parser/src/hir/syntax_shape/expression.rs @@ -4,6 +4,7 @@ pub(crate) mod file_path; pub(crate) mod list; pub(crate) mod number; pub(crate) mod pattern; +pub(crate) mod range; pub(crate) mod string; pub(crate) mod unit; pub(crate) mod variable_path; diff --git a/crates/nu-parser/src/hir/syntax_shape/expression/atom.rs b/crates/nu-parser/src/hir/syntax_shape/expression/atom.rs index 6fe8a992ac..689c69bdd9 100644 --- a/crates/nu-parser/src/hir/syntax_shape/expression/atom.rs +++ b/crates/nu-parser/src/hir/syntax_shape/expression/atom.rs @@ -3,6 +3,7 @@ use crate::hir::syntax_shape::{ expand_syntax, expression::expand_file_path, parse_single_node, BarePathShape, BarePatternShape, ExpandContext, UnitShape, UnitSyntax, }; +use crate::parse::operator::EvaluationOperator; use crate::parse::token_tree::{DelimitedNode, Delimiter, TokenNode}; use crate::parse::tokens::UnspannedToken; use crate::parse::unit::Unit; @@ -12,7 +13,7 @@ use crate::{ parse::flag::{Flag, FlagKind}, }; use nu_errors::{ParseError, ShellError}; -use nu_protocol::ShellTypeName; +use nu_protocol::{ShellTypeName, SpannedTypeName}; use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span, Spanned, SpannedItem}; use std::ops::Deref; @@ -52,23 +53,36 @@ pub enum UnspannedAtomicToken<'tokens> { Word { text: Span, }, - #[allow(unused)] - Dot { - text: Span, - }, SquareDelimited { spans: (Span, Span), nodes: &'tokens Vec, }, + #[allow(unused)] + RoundDelimited { + spans: (Span, Span), + nodes: &'tokens Vec, + }, ShorthandFlag { name: Span, }, - Operator { + CompareOperator { + text: Span, + }, + Dot { + text: Span, + }, + DotDot { text: Span, }, Whitespace { text: Span, }, + Separator { + text: Span, + }, + Comment { + body: Span, + }, } impl<'tokens> UnspannedAtomicToken<'tokens> { @@ -80,15 +94,24 @@ impl<'tokens> UnspannedAtomicToken<'tokens> { } } +impl<'tokens> ShellTypeName for AtomicToken<'tokens> { + fn type_name(&self) -> &'static str { + self.unspanned.type_name() + } +} + impl<'tokens> ShellTypeName for UnspannedAtomicToken<'tokens> { fn type_name(&self) -> &'static str { match &self { UnspannedAtomicToken::Eof { .. } => "eof", UnspannedAtomicToken::Error { .. } => "error", - UnspannedAtomicToken::Operator { .. } => "operator", + UnspannedAtomicToken::CompareOperator { .. } => "compare operator", UnspannedAtomicToken::ShorthandFlag { .. } => "shorthand flag", UnspannedAtomicToken::Whitespace { .. } => "whitespace", + UnspannedAtomicToken::Separator { .. } => "separator", + UnspannedAtomicToken::Comment { .. } => "comment", UnspannedAtomicToken::Dot { .. } => "dot", + UnspannedAtomicToken::DotDot { .. } => "dotdot", UnspannedAtomicToken::Number { .. } => "number", UnspannedAtomicToken::Size { .. } => "size", UnspannedAtomicToken::String { .. } => "string", @@ -99,6 +122,7 @@ impl<'tokens> ShellTypeName for UnspannedAtomicToken<'tokens> { UnspannedAtomicToken::GlobPattern { .. } => "file pattern", UnspannedAtomicToken::Word { .. } => "word", UnspannedAtomicToken::SquareDelimited { .. } => "array literal", + UnspannedAtomicToken::RoundDelimited { .. } => "paren delimited", } } } @@ -109,6 +133,12 @@ pub struct AtomicToken<'tokens> { pub span: Span, } +impl<'tokens> HasSpan for AtomicToken<'tokens> { + fn span(&self) -> Span { + self.span + } +} + impl<'tokens> Deref for AtomicToken<'tokens> { type Target = UnspannedAtomicToken<'tokens>; @@ -131,31 +161,18 @@ impl<'tokens> AtomicToken<'tokens> { )) } UnspannedAtomicToken::Error { .. } => { - return Err(ParseError::mismatch( - expected, - "eof atomic token".spanned(self.span), - )) + return Err(ParseError::mismatch(expected, "error".spanned(self.span))) } - UnspannedAtomicToken::Operator { .. } => { - return Err(ParseError::mismatch( - expected, - "operator".spanned(self.span), - )) - } - UnspannedAtomicToken::ShorthandFlag { .. } => { - return Err(ParseError::mismatch( - expected, - "shorthand flag".spanned(self.span), - )) - } - UnspannedAtomicToken::Whitespace { .. } => { - return Err(ParseError::mismatch( - expected, - "whitespace".spanned(self.span), - )) - } - UnspannedAtomicToken::Dot { .. } => { - return Err(ParseError::mismatch(expected, "dot".spanned(self.span))) + UnspannedAtomicToken::RoundDelimited { .. } + | UnspannedAtomicToken::CompareOperator { .. } + | UnspannedAtomicToken::ShorthandFlag { .. } + | UnspannedAtomicToken::Whitespace { .. } + | UnspannedAtomicToken::Separator { .. } + | UnspannedAtomicToken::Comment { .. } + | UnspannedAtomicToken::Dot { .. } + | UnspannedAtomicToken::DotDot { .. } + | UnspannedAtomicToken::SquareDelimited { .. } => { + return Err(ParseError::mismatch(expected, self.spanned_type_name())); } UnspannedAtomicToken::Number { number } => { Expression::number(number.to_number(context.source), self.span) @@ -175,41 +192,17 @@ impl<'tokens> AtomicToken<'tokens> { self.span, ), UnspannedAtomicToken::Word { text } => Expression::string(*text, *text), - UnspannedAtomicToken::SquareDelimited { .. } => unimplemented!("into_hir"), }) } - #[cfg(not(coloring_in_tokens))] - pub fn spanned_type_name(&self) -> Spanned<&'static str> { - match &self.unspanned { - UnspannedAtomicToken::Eof { .. } => "eof", - UnspannedAtomicToken::Error { .. } => "error", - UnspannedAtomicToken::Operator { .. } => "operator", - UnspannedAtomicToken::ShorthandFlag { .. } => "shorthand flag", - UnspannedAtomicToken::Whitespace { .. } => "whitespace", - UnspannedAtomicToken::Dot { .. } => "dot", - UnspannedAtomicToken::Number { .. } => "number", - UnspannedAtomicToken::Size { .. } => "size", - UnspannedAtomicToken::String { .. } => "string", - UnspannedAtomicToken::ItVariable { .. } => "$it", - UnspannedAtomicToken::Variable { .. } => "variable", - UnspannedAtomicToken::ExternalCommand { .. } => "external command", - UnspannedAtomicToken::ExternalWord { .. } => "external word", - UnspannedAtomicToken::GlobPattern { .. } => "file pattern", - UnspannedAtomicToken::Word { .. } => "word", - UnspannedAtomicToken::SquareDelimited { .. } => "array literal", - } - .spanned(self.span) - } - pub(crate) fn color_tokens(&self, shapes: &mut Vec>) { match &self.unspanned { UnspannedAtomicToken::Eof { .. } => {} UnspannedAtomicToken::Error { .. } => { return shapes.push(FlatShape::Error.spanned(self.span)) } - UnspannedAtomicToken::Operator { .. } => { - return shapes.push(FlatShape::Operator.spanned(self.span)); + UnspannedAtomicToken::CompareOperator { .. } => { + return shapes.push(FlatShape::CompareOperator.spanned(self.span)); } UnspannedAtomicToken::ShorthandFlag { .. } => { return shapes.push(FlatShape::ShorthandFlag.spanned(self.span)); @@ -305,17 +298,30 @@ impl PrettyDebugWithSource for AtomicToken<'_> { b::intersperse_with_source(nodes.iter(), b::space(), source), "]", ), + UnspannedAtomicToken::RoundDelimited { nodes, .. } => b::delimit( + "(", + b::intersperse_with_source(nodes.iter(), b::space(), source), + ")", + ), UnspannedAtomicToken::ShorthandFlag { name } => { atom_kind("shorthand flag", b::key(name.slice(source))) } UnspannedAtomicToken::Dot { .. } => atom(b::kind("dot")), - UnspannedAtomicToken::Operator { text } => { + UnspannedAtomicToken::DotDot { .. } => atom(b::kind("dotdot")), + UnspannedAtomicToken::CompareOperator { text } => { atom_kind("operator", b::keyword(text.slice(source))) } UnspannedAtomicToken::Whitespace { text } => atom_kind( "whitespace", b::description(format!("{:?}", text.slice(source))), ), + UnspannedAtomicToken::Separator { text } => atom_kind( + "separator", + b::description(format!("{:?}", text.slice(source))), + ), + UnspannedAtomicToken::Comment { body } => { + atom_kind("comment", b::description(body.slice(source))) + } }) } } @@ -331,12 +337,15 @@ pub enum WhitespaceHandling { pub struct ExpansionRule { pub(crate) allow_external_command: bool, pub(crate) allow_external_word: bool, - pub(crate) allow_operator: bool, + pub(crate) allow_cmp_operator: bool, + pub(crate) allow_eval_operator: bool, pub(crate) allow_eof: bool, + pub(crate) allow_separator: bool, pub(crate) treat_size_as_word: bool, pub(crate) separate_members: bool, pub(crate) commit_errors: bool, pub(crate) whitespace: WhitespaceHandling, + pub(crate) allow_comments: bool, } impl ExpansionRule { @@ -344,12 +353,15 @@ impl ExpansionRule { ExpansionRule { allow_external_command: false, allow_external_word: false, - allow_operator: false, + allow_eval_operator: false, + allow_cmp_operator: false, allow_eof: false, treat_size_as_word: false, separate_members: false, commit_errors: false, + allow_separator: false, whitespace: WhitespaceHandling::RejectWhitespace, + allow_comments: false, } } @@ -360,11 +372,14 @@ impl ExpansionRule { ExpansionRule { allow_external_command: true, allow_external_word: true, - allow_operator: true, + allow_cmp_operator: true, + allow_eval_operator: true, allow_eof: true, separate_members: false, treat_size_as_word: false, commit_errors: true, + allow_separator: true, + allow_comments: true, whitespace: WhitespaceHandling::AllowWhitespace, } } @@ -376,14 +391,26 @@ impl ExpansionRule { } #[allow(unused)] - pub fn allow_operator(mut self) -> ExpansionRule { - self.allow_operator = true; + pub fn allow_cmp_operator(mut self) -> ExpansionRule { + self.allow_cmp_operator = true; + self + } + + #[allow(unused)] + pub fn no_cmp_operator(mut self) -> ExpansionRule { + self.allow_cmp_operator = false; + self + } + + #[allow(unused)] + pub fn allow_eval_operator(mut self) -> ExpansionRule { + self.allow_eval_operator = true; self } #[allow(unused)] pub fn no_operator(mut self) -> ExpansionRule { - self.allow_operator = false; + self.allow_eval_operator = false; self } @@ -440,6 +467,30 @@ impl ExpansionRule { self.whitespace = WhitespaceHandling::RejectWhitespace; self } + + #[allow(unused)] + pub fn allow_separator(mut self) -> ExpansionRule { + self.allow_separator = true; + self + } + + #[allow(unused)] + pub fn reject_separator(mut self) -> ExpansionRule { + self.allow_separator = false; + self + } + + #[allow(unused)] + pub fn allow_comments(mut self) -> ExpansionRule { + self.allow_comments = true; + self + } + + #[allow(unused)] + pub fn reject_comments(mut self) -> ExpansionRule { + self.allow_comments = false; + self + } } pub fn expand_atom<'me, 'content>( @@ -578,6 +629,17 @@ fn expand_atom_inner<'me, 'content>( .into_atomic_token(error.span)); } + TokenNode::Separator(span) if rule.allow_separator => { + peeked.commit(); + return Ok(UnspannedAtomicToken::Separator { text: *span }.into_atomic_token(span)); + } + + TokenNode::Comment(comment) if rule.allow_comments => { + peeked.commit(); + return Ok(UnspannedAtomicToken::Comment { body: comment.text } + .into_atomic_token(comment.span())); + } + // [ ... ] TokenNode::Delimited(Spanned { item: @@ -649,8 +711,16 @@ fn expand_atom_inner<'me, 'content>( // First, the error cases. Each error case corresponds to a expansion rule // flag that can be used to allow the case - // rule.allow_operator - UnspannedToken::Operator(_) if !rule.allow_operator => return Err(err.error()), + // rule.allow_cmp_operator + UnspannedToken::CompareOperator(_) if !rule.allow_cmp_operator => { + return Err(err.error()) + } + + // rule.allow_eval_operator + UnspannedToken::EvaluationOperator(_) if !rule.allow_eval_operator => { + return Err(err.error()) + } + // rule.allow_external_command UnspannedToken::ExternalCommand(_) if !rule.allow_external_command => { return Err(ParseError::mismatch( @@ -669,8 +739,15 @@ fn expand_atom_inner<'me, 'content>( UnspannedToken::Number(number) => { UnspannedAtomicToken::Number { number }.into_atomic_token(token_span) } - UnspannedToken::Operator(_) => { - UnspannedAtomicToken::Operator { text: token_span }.into_atomic_token(token_span) + UnspannedToken::CompareOperator(_) => { + UnspannedAtomicToken::CompareOperator { text: token_span } + .into_atomic_token(token_span) + } + UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => { + UnspannedAtomicToken::Dot { text: token_span }.into_atomic_token(token_span) + } + UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot) => { + UnspannedAtomicToken::DotDot { text: token_span }.into_atomic_token(token_span) } UnspannedToken::String(body) => { UnspannedAtomicToken::String { body }.into_atomic_token(token_span) diff --git a/crates/nu-parser/src/hir/syntax_shape/expression/number.rs b/crates/nu-parser/src/hir/syntax_shape/expression/number.rs index ed95439ce2..22494dd6a0 100644 --- a/crates/nu-parser/src/hir/syntax_shape/expression/number.rs +++ b/crates/nu-parser/src/hir/syntax_shape/expression/number.rs @@ -26,9 +26,9 @@ impl ExpandExpression for NumberShape { ) -> Result { parse_single_node(token_nodes, "Number", |token, token_span, err| { Ok(match token { - UnspannedToken::GlobPattern | UnspannedToken::Operator(..) => { - return Err(err.error()) - } + UnspannedToken::GlobPattern + | UnspannedToken::CompareOperator(..) + | UnspannedToken::EvaluationOperator(..) => return Err(err.error()), UnspannedToken::Variable(tag) if tag.slice(context.source) == "it" => { hir::Expression::it_variable(tag, token_span) } @@ -131,7 +131,8 @@ impl ExpandExpression for IntShape { parse_single_node(token_nodes, "Integer", |token, token_span, err| { Ok(match token { UnspannedToken::GlobPattern - | UnspannedToken::Operator(..) + | UnspannedToken::CompareOperator(..) + | UnspannedToken::EvaluationOperator(..) | UnspannedToken::ExternalWord => return Err(err.error()), UnspannedToken::Variable(span) if span.slice(context.source) == "it" => { hir::Expression::it_variable(span, token_span) diff --git a/crates/nu-parser/src/hir/syntax_shape/expression/pattern.rs b/crates/nu-parser/src/hir/syntax_shape/expression/pattern.rs index 826d5d8286..96f7d9da9d 100644 --- a/crates/nu-parser/src/hir/syntax_shape/expression/pattern.rs +++ b/crates/nu-parser/src/hir/syntax_shape/expression/pattern.rs @@ -2,8 +2,9 @@ use crate::hir::syntax_shape::{ expand_atom, expand_bare, expression::expand_file_path, ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, UnspannedAtomicToken, }; +use crate::parse::operator::EvaluationOperator; use crate::parse::tokens::{Token, UnspannedToken}; -use crate::{hir, hir::TokensIterator, Operator, TokenNode}; +use crate::{hir, hir::TokensIterator, TokenNode}; use nu_errors::{ParseError, ShellError}; #[cfg(coloring_in_tokens)] use nu_protocol::ShellTypeName; @@ -26,6 +27,8 @@ impl FallibleColorSyntax for PatternShape { context: &ExpandContext, shapes: &mut Vec>, ) -> Result<(), ShellError> { + use nu_protocol::SpannedTypeName; + token_nodes.atomic(|token_nodes| { let atom = expand_atom(token_nodes, "pattern", context, ExpansionRule::permissive())?; @@ -125,7 +128,7 @@ impl ExpandSyntax for BarePatternShape { .. }) | TokenNode::Token(Token { - unspanned: UnspannedToken::Operator(Operator::Dot), + unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::Dot), .. }) | TokenNode::Token(Token { diff --git a/crates/nu-parser/src/hir/syntax_shape/expression/range.rs b/crates/nu-parser/src/hir/syntax_shape/expression/range.rs new file mode 100644 index 0000000000..ca32db12f7 --- /dev/null +++ b/crates/nu-parser/src/hir/syntax_shape/expression/range.rs @@ -0,0 +1,154 @@ +use crate::hir::syntax_shape::expression::UnspannedAtomicToken; +use crate::hir::syntax_shape::{ + color_fallible_syntax, expand_atom, expand_expr, AnyExpressionShape, ExpandContext, + ExpandExpression, ExpansionRule, FallibleColorSyntax, FlatShape, +}; +use crate::parse::operator::EvaluationOperator; +use crate::parse::token_tree::TokenNode; +use crate::parse::tokens::{Token, UnspannedToken}; +use crate::{hir, hir::TokensIterator}; +use nu_errors::{ParseError, ShellError}; +use nu_protocol::SpannedTypeName; +use nu_source::SpannedItem; + +#[derive(Debug, Copy, Clone)] +pub struct RangeShape; + +impl ExpandExpression for RangeShape { + fn name(&self) -> &'static str { + "range" + } + + fn expand_expr<'a, 'b>( + &self, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result { + token_nodes.atomic_parse(|token_nodes| { + let left = expand_expr(&AnyExpressionShape, token_nodes, context)?; + + let atom = expand_atom( + token_nodes, + "..", + context, + ExpansionRule::new().allow_eval_operator(), + )?; + + let span = match atom.unspanned { + UnspannedAtomicToken::DotDot { text } => text, + _ => return Err(ParseError::mismatch("..", atom.spanned_type_name())), + }; + + let right = expand_expr(&AnyExpressionShape, token_nodes, context)?; + + Ok(hir::Expression::range(left, span, right)) + }) + } +} + +#[cfg(coloring_in_tokens)] +impl FallibleColorSyntax for RangeShape { + type Info = (); + type Input = (); + + fn name(&self) -> &'static str { + "RangeShape" + } + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + ) -> Result<(), ShellError> { + token_nodes.atomic_parse(|token_nodes| { + color_fallible_syntax(&AnyExpressionShape, token_nodes, context)?; + color_fallible_syntax(&DotDotShape, token_nodes, context)?; + color_fallible_syntax(&AnyExpressionShape, token_nodes, context) + })?; + + Ok(()) + } +} + +#[cfg(not(coloring_in_tokens))] +impl FallibleColorSyntax for RangeShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &(), + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result<(), ShellError> { + token_nodes.atomic_parse(|token_nodes| { + color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes)?; + color_fallible_syntax(&DotDotShape, token_nodes, context, shapes)?; + color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes) + })?; + + Ok(()) + } +} + +#[derive(Debug, Copy, Clone)] +struct DotDotShape; + +#[cfg(coloring_in_tokens)] +impl FallibleColorSyntax for DotDotShape { + type Info = (); + type Input = (); + + fn name(&self) -> &'static str { + ".." + } + + fn color_syntax<'a, 'b>( + &self, + _input: &Self::Input, + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + ) -> Result { + let peeked = token_nodes.peek_any().not_eof("..")?; + match &peeked.node { + TokenNode::Token(Token { + unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot), + span, + }) => { + peeked.commit(); + token_nodes.color_shape(FlatShape::DotDot.spanned(span)); + Ok(()) + } + token => Err(ShellError::type_error("..", token.spanned_type_name())), + } + } +} + +#[cfg(not(coloring_in_tokens))] +impl FallibleColorSyntax for DotDotShape { + type Info = (); + type Input = (); + + fn color_syntax<'a, 'b>( + &self, + _input: &Self::Input, + token_nodes: &'b mut TokensIterator<'a>, + _context: &ExpandContext, + shapes: &mut Vec>, + ) -> Result { + let peeked = token_nodes.peek_any().not_eof("..")?; + match &peeked.node { + TokenNode::Token(Token { + unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot), + span, + }) => { + peeked.commit(); + shapes.push(FlatShape::DotDot.spanned(span)); + Ok(()) + } + token => Err(ShellError::type_error("..", token.spanned_type_name())), + } + } +} diff --git a/crates/nu-parser/src/hir/syntax_shape/expression/string.rs b/crates/nu-parser/src/hir/syntax_shape/expression/string.rs index caf9b0abad..7844a9174a 100644 --- a/crates/nu-parser/src/hir/syntax_shape/expression/string.rs +++ b/crates/nu-parser/src/hir/syntax_shape/expression/string.rs @@ -91,7 +91,8 @@ impl ExpandExpression for StringShape { parse_single_node(token_nodes, "String", |token, token_span, err| { Ok(match token { UnspannedToken::GlobPattern - | UnspannedToken::Operator(..) + | UnspannedToken::CompareOperator(..) + | UnspannedToken::EvaluationOperator(..) | UnspannedToken::ExternalWord => return Err(err.error()), UnspannedToken::Variable(span) => { expand_variable(span, token_span, &context.source) diff --git a/crates/nu-parser/src/hir/syntax_shape/expression/variable_path.rs b/crates/nu-parser/src/hir/syntax_shape/expression/variable_path.rs index 2f075a9d6f..e0173899b6 100644 --- a/crates/nu-parser/src/hir/syntax_shape/expression/variable_path.rs +++ b/crates/nu-parser/src/hir/syntax_shape/expression/variable_path.rs @@ -5,7 +5,7 @@ use crate::hir::syntax_shape::{ StringShape, TestSyntax, UnspannedAtomicToken, WhitespaceShape, }; use crate::parse::tokens::{RawNumber, UnspannedToken}; -use crate::{hir, hir::Expression, hir::TokensIterator, Operator}; +use crate::{hir, hir::Expression, hir::TokensIterator, CompareOperator, EvaluationOperator}; use nu_errors::ShellError; use nu_protocol::{PathMember, ShellTypeName}; use nu_source::{ @@ -271,7 +271,7 @@ impl ExpandSyntax for PathTailShape { #[derive(Debug, Clone)] pub enum ExpressionContinuation { DotSuffix(Span, PathMember), - InfixSuffix(Spanned, Expression), + InfixSuffix(Spanned, Expression), } impl PrettyDebugWithSource for ExpressionContinuation { @@ -484,6 +484,8 @@ impl FallibleColorSyntax for VariableShape { context: &ExpandContext, shapes: &mut Vec>, ) -> Result<(), ShellError> { + use nu_protocol::SpannedTypeName; + let atom = expand_atom( token_nodes, "variable", @@ -1032,6 +1034,8 @@ impl FallibleColorSyntax for ColorableDotShape { _context: &ExpandContext, shapes: &mut Vec>, ) -> Result<(), ShellError> { + use nu_protocol::SpannedTypeName; + let peeked = token_nodes.peek_any().not_eof("dot")?; match peeked.node { @@ -1104,7 +1108,7 @@ impl ExpandSyntax for DotShape { ) -> Result { parse_single_node(token_nodes, "dot", |token, token_span, _| { Ok(match token { - UnspannedToken::Operator(Operator::Dot) => token_span, + UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => token_span, _ => { return Err(ParseError::mismatch( "dot", @@ -1143,9 +1147,9 @@ impl FallibleColorSyntax for InfixShape { "infix operator", |token, token_span, err| { match token { - // If it's an operator (and not `.`), it's a match - UnspannedToken::Operator(operator) if operator != Operator::Dot => { - shapes.push(FlatShape::Operator.spanned(token_span)); + // If it's a comparison operator, it's a match + UnspannedToken::CompareOperator(_operator) => { + shapes.push(FlatShape::CompareOperator.spanned(token_span)); Ok(()) } @@ -1191,9 +1195,7 @@ impl FallibleColorSyntax for InfixShape { |token, token_span, _| { match token { // If it's an operator (and not `.`), it's a match - UnspannedToken::Operator(operator) if operator != Operator::Dot => { - Ok(token_span) - } + UnspannedToken::CompareOperator(_operator) => Ok(token_span), // Otherwise, it's not a match _ => Err(ParseError::mismatch( @@ -1206,7 +1208,7 @@ impl FallibleColorSyntax for InfixShape { checkpoint .iterator - .color_shape(FlatShape::Operator.spanned(operator_span)); + .color_shape(FlatShape::CompareOperator.spanned(operator_span)); // An infix operator must be followed by whitespace. If no whitespace was found, fail color_fallible_syntax(&WhitespaceShape, checkpoint.iterator, context)?; @@ -1266,7 +1268,7 @@ impl ExpandSyntax for InfixShape { #[derive(Debug, Clone)] pub struct InfixInnerSyntax { - pub operator: Spanned, + pub operator: Spanned, } impl HasSpan for InfixInnerSyntax { @@ -1298,12 +1300,10 @@ impl ExpandSyntax for InfixInnerShape { ) -> Result { parse_single_node(token_nodes, "infix operator", |token, token_span, err| { Ok(match token { - // If it's an operator (and not `.`), it's a match - UnspannedToken::Operator(operator) if operator != Operator::Dot => { - InfixInnerSyntax { - operator: operator.spanned(token_span), - } - } + // If it's a comparison operator, it's a match + UnspannedToken::CompareOperator(operator) => InfixInnerSyntax { + operator: operator.spanned(token_span), + }, // Otherwise, it's not a match _ => return Err(err.error()), diff --git a/crates/nu-parser/src/hir/syntax_shape/flat_shape.rs b/crates/nu-parser/src/hir/syntax_shape/flat_shape.rs index 2f2c865652..6c125ea547 100644 --- a/crates/nu-parser/src/hir/syntax_shape/flat_shape.rs +++ b/crates/nu-parser/src/hir/syntax_shape/flat_shape.rs @@ -1,5 +1,5 @@ use crate::parse::flag::{Flag, FlagKind}; -use crate::parse::operator::Operator; +use crate::parse::operator::EvaluationOperator; use crate::parse::token_tree::{Delimiter, TokenNode}; use crate::parse::tokens::{RawNumber, UnspannedToken}; use nu_source::{HasSpan, Span, Spanned, SpannedItem, Text}; @@ -10,8 +10,9 @@ pub enum FlatShape { CloseDelimiter(Delimiter), ItVariable, Variable, - Operator, + CompareOperator, Dot, + DotDot, InternalCommand, ExternalCommand, ExternalWord, @@ -27,7 +28,9 @@ pub enum FlatShape { Int, Decimal, Whitespace, + Separator, Error, + Comment, Size { number: Span, unit: Span }, } @@ -41,10 +44,15 @@ impl FlatShape { UnspannedToken::Number(RawNumber::Decimal(_)) => { shapes.push(FlatShape::Decimal.spanned(token.span)) } - UnspannedToken::Operator(Operator::Dot) => { + UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => { shapes.push(FlatShape::Dot.spanned(token.span)) } - UnspannedToken::Operator(_) => shapes.push(FlatShape::Operator.spanned(token.span)), + UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot) => { + shapes.push(FlatShape::DotDot.spanned(token.span)) + } + UnspannedToken::CompareOperator(_) => { + shapes.push(FlatShape::CompareOperator.spanned(token.span)) + } UnspannedToken::String(_) => shapes.push(FlatShape::String.spanned(token.span)), UnspannedToken::Variable(v) if v.slice(source) == "it" => { shapes.push(FlatShape::ItVariable.spanned(token.span)) @@ -92,6 +100,8 @@ impl FlatShape { .. }) => shapes.push(FlatShape::ShorthandFlag.spanned(*span)), TokenNode::Whitespace(_) => shapes.push(FlatShape::Whitespace.spanned(token.span())), + TokenNode::Separator(_) => shapes.push(FlatShape::Separator.spanned(token.span())), + TokenNode::Comment(_) => shapes.push(FlatShape::Comment.spanned(token.span())), TokenNode::Error(v) => shapes.push(FlatShape::Error.spanned(v.span)), } } diff --git a/crates/nu-parser/src/hir/tokens_iterator.rs b/crates/nu-parser/src/hir/tokens_iterator.rs index 49ddd25a9b..a28b34d866 100644 --- a/crates/nu-parser/src/hir/tokens_iterator.rs +++ b/crates/nu-parser/src/hir/tokens_iterator.rs @@ -8,6 +8,7 @@ use crate::TokenNode; #[allow(unused)] use getset::{Getters, MutGetters}; use nu_errors::{ParseError, ShellError}; +use nu_protocol::SpannedTypeName; use nu_source::{HasFallibleSpan, HasSpan, Span, Spanned, SpannedItem, Tag, Text}; cfg_if::cfg_if! { @@ -149,7 +150,7 @@ impl<'content, 'me> PeekedNode<'content, 'me> { pub fn peek_error(node: &Option<&TokenNode>, eof_span: Span, expected: &'static str) -> ParseError { match node { None => ParseError::unexpected_eof(expected, eof_span), - Some(node) => ParseError::mismatch(expected, node.type_name().spanned(node.span())), + Some(node) => ParseError::mismatch(expected, node.spanned_type_name()), } } @@ -498,10 +499,10 @@ impl<'content> TokensIterator<'content> { /// Use a checkpoint when you need to peek more than one token ahead, but can't be sure /// that you'll succeed. - pub fn atomic_parse<'me, T>( + pub fn atomic_parse<'me, T, E>( &'me mut self, - block: impl FnOnce(&mut TokensIterator<'content>) -> Result, - ) -> Result { + block: impl FnOnce(&mut TokensIterator<'content>) -> Result, + ) -> Result { let state = &mut self.state; let index = state.index; diff --git a/crates/nu-parser/src/lib.rs b/crates/nu-parser/src/lib.rs index 05194a67eb..f1e8baca90 100644 --- a/crates/nu-parser/src/lib.rs +++ b/crates/nu-parser/src/lib.rs @@ -12,9 +12,9 @@ pub use crate::hir::syntax_shape::{ pub use crate::hir::tokens_iterator::TokensIterator; pub use crate::parse::files::Files; pub use crate::parse::flag::Flag; -pub use crate::parse::operator::Operator; -pub use crate::parse::parser::pipeline; +pub use crate::parse::operator::{CompareOperator, EvaluationOperator}; pub use crate::parse::parser::Number; +pub use crate::parse::parser::{module, pipeline}; pub use crate::parse::token_tree::{Delimiter, TokenNode}; pub use crate::parse::token_tree_builder::TokenTreeBuilder; @@ -29,3 +29,12 @@ pub fn parse(input: &str) -> Result { Err(err) => Err(ShellError::parse_error(err)), } } + +pub fn parse_script(input: &str) -> Result { + let _ = pretty_env_logger::try_init(); + + match module(nom_input(input)) { + Ok((_rest, val)) => Ok(val), + Err(err) => Err(ShellError::parse_error(err)), + } +} diff --git a/crates/nu-parser/src/parse.rs b/crates/nu-parser/src/parse.rs index 6f9fac0b63..7528b99983 100644 --- a/crates/nu-parser/src/parse.rs +++ b/crates/nu-parser/src/parse.rs @@ -1,4 +1,5 @@ pub(crate) mod call_node; +pub(crate) mod comment; pub(crate) mod files; pub(crate) mod flag; pub(crate) mod operator; diff --git a/crates/nu-parser/src/parse/comment.rs b/crates/nu-parser/src/parse/comment.rs new file mode 100644 index 0000000000..57d26f338c --- /dev/null +++ b/crates/nu-parser/src/parse/comment.rs @@ -0,0 +1,42 @@ +use derive_new::new; +use getset::Getters; +use nu_source::{b, DebugDocBuilder, HasSpan, PrettyDebugWithSource, Span}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] +pub enum CommentKind { + Line, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Getters, new)] +pub struct Comment { + pub(crate) kind: CommentKind, + pub(crate) text: Span, + pub(crate) span: Span, +} + +impl Comment { + pub fn line(text: impl Into, outer: impl Into) -> Comment { + Comment { + kind: CommentKind::Line, + text: text.into(), + span: outer.into(), + } + } +} + +impl PrettyDebugWithSource for Comment { + fn pretty_debug(&self, source: &str) -> DebugDocBuilder { + let prefix = match self.kind { + CommentKind::Line => b::description("#"), + }; + + prefix + b::description(self.text.slice(source)) + } +} + +impl HasSpan for Comment { + fn span(&self) -> Span { + self.span + } +} diff --git a/crates/nu-parser/src/parse/files.rs b/crates/nu-parser/src/parse/files.rs index f472b59972..4df5d48fd4 100644 --- a/crates/nu-parser/src/parse/files.rs +++ b/crates/nu-parser/src/parse/files.rs @@ -34,13 +34,15 @@ impl language_reporting::ReportingFiles for Files { } fn location(&self, _file: Self::FileId, byte_index: usize) -> Option { + trace!("finding location for {}", byte_index); + let source = &self.snippet; let mut seen_lines = 0; let mut seen_bytes = 0; for (pos, slice) in source.match_indices('\n') { trace!( - "SEARCH={} SEEN={} POS={} SLICE={:?} LEN={} ALL={:?}", + "searching byte_index={} seen_bytes={} pos={} slice={:?} slice.len={} source={:?}", byte_index, seen_bytes, pos, @@ -50,9 +52,19 @@ impl language_reporting::ReportingFiles for Files { ); if pos >= byte_index { + trace!( + "returning {}:{} seen_lines={} byte_index={} pos={} seen_bytes={}", + seen_lines, + byte_index, + pos, + seen_lines, + byte_index, + seen_bytes + ); + return Some(language_reporting::Location::new( seen_lines, - byte_index - seen_bytes, + byte_index - pos, )); } else { seen_lines += 1; @@ -61,30 +73,70 @@ impl language_reporting::ReportingFiles for Files { } if seen_lines == 0 { - Some(language_reporting::Location::new(0, byte_index)) + trace!("seen_lines=0 end={}", source.len() - 1); + + // if we got here, there were no newlines in the source + Some(language_reporting::Location::new(0, source.len() - 1)) } else { - panic!("byte index {} wasn't valid", byte_index); + trace!( + "last line seen_lines={} end={}", + seen_lines, + source.len() - 1 - byte_index + ); + + // if we got here and we didn't return, it should mean that we're talking about + // the last line + Some(language_reporting::Location::new( + seen_lines, + source.len() - 1 - byte_index, + )) } } fn line_span(&self, _file: Self::FileId, lineno: usize) -> Option { + trace!("finding line_span for {}", lineno); + let source = &self.snippet; let mut seen_lines = 0; let mut seen_bytes = 0; for (pos, _) in source.match_indices('\n') { + trace!( + "lineno={} seen_lines={} seen_bytes={} pos={}", + lineno, + seen_lines, + seen_bytes, + pos + ); + if seen_lines == lineno { - return Some(Span::new(seen_bytes, pos + 1)); + trace!("returning start={} end={}", seen_bytes, pos); + // If the number of seen lines is the lineno, seen_bytes is the start of the + // line and pos is the end of the line + return Some(Span::new(seen_bytes, pos)); } else { + // If it's not, increment seen_lines, and move seen_bytes to the beginning of + // the next line seen_lines += 1; seen_bytes = pos + 1; } } if seen_lines == 0 { + trace!("returning start={} end={}", 0, self.snippet.len() - 1); + + // if we got here, there were no newlines in the source Some(Span::new(0, self.snippet.len() - 1)) } else { - None + trace!( + "returning start={} end={}", + seen_bytes, + self.snippet.len() - 1 + ); + + // if we got here and we didn't return, it should mean that we're talking about + // the last line + Some(Span::new(seen_bytes, self.snippet.len() - 1)) } } diff --git a/crates/nu-parser/src/parse/operator.rs b/crates/nu-parser/src/parse/operator.rs index 99e5b5499e..a5504a6195 100644 --- a/crates/nu-parser/src/parse/operator.rs +++ b/crates/nu-parser/src/parse/operator.rs @@ -4,63 +4,102 @@ use serde::{Deserialize, Serialize}; use std::str::FromStr; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] -pub enum Operator { +pub enum CompareOperator { Equal, NotEqual, LessThan, GreaterThan, LessThanOrEqual, GreaterThanOrEqual, - Dot, Contains, NotContains, } -impl PrettyDebug for Operator { +impl PrettyDebug for CompareOperator { fn pretty(&self) -> DebugDocBuilder { b::operator(self.as_str()) } } -impl Operator { +impl CompareOperator { pub fn print(&self) -> String { self.as_str().to_string() } pub fn as_str(&self) -> &str { match *self { - Operator::Equal => "==", - Operator::NotEqual => "!=", - Operator::LessThan => "<", - Operator::GreaterThan => ">", - Operator::LessThanOrEqual => "<=", - Operator::GreaterThanOrEqual => ">=", - Operator::Dot => ".", - Operator::Contains => "=~", - Operator::NotContains => "!~", + CompareOperator::Equal => "==", + CompareOperator::NotEqual => "!=", + CompareOperator::LessThan => "<", + CompareOperator::GreaterThan => ">", + CompareOperator::LessThanOrEqual => "<=", + CompareOperator::GreaterThanOrEqual => ">=", + CompareOperator::Contains => "=~", + CompareOperator::NotContains => "!~", } } } -impl From<&str> for Operator { - fn from(input: &str) -> Operator { - Operator::from_str(input).unwrap() +impl From<&str> for CompareOperator { + fn from(input: &str) -> CompareOperator { + CompareOperator::from_str(input).unwrap() } } -impl FromStr for Operator { +impl FromStr for CompareOperator { type Err = (); fn from_str(input: &str) -> Result::Err> { match input { - "==" => Ok(Operator::Equal), - "!=" => Ok(Operator::NotEqual), - "<" => Ok(Operator::LessThan), - ">" => Ok(Operator::GreaterThan), - "<=" => Ok(Operator::LessThanOrEqual), - ">=" => Ok(Operator::GreaterThanOrEqual), - "." => Ok(Operator::Dot), - "=~" => Ok(Operator::Contains), - "!~" => Ok(Operator::NotContains), + "==" => Ok(CompareOperator::Equal), + "!=" => Ok(CompareOperator::NotEqual), + "<" => Ok(CompareOperator::LessThan), + ">" => Ok(CompareOperator::GreaterThan), + "<=" => Ok(CompareOperator::LessThanOrEqual), + ">=" => Ok(CompareOperator::GreaterThanOrEqual), + "=~" => Ok(CompareOperator::Contains), + "!~" => Ok(CompareOperator::NotContains), + _ => Err(()), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Deserialize, Serialize)] +pub enum EvaluationOperator { + Dot, + DotDot, +} + +impl PrettyDebug for EvaluationOperator { + fn pretty(&self) -> DebugDocBuilder { + b::operator(self.as_str()) + } +} + +impl EvaluationOperator { + pub fn print(&self) -> String { + self.as_str().to_string() + } + + pub fn as_str(&self) -> &str { + match *self { + EvaluationOperator::Dot => ".", + EvaluationOperator::DotDot => "..", + } + } +} + +impl From<&str> for EvaluationOperator { + fn from(input: &str) -> EvaluationOperator { + EvaluationOperator::from_str(input).unwrap() + } +} + +impl FromStr for EvaluationOperator { + type Err = (); + fn from_str(input: &str) -> Result::Err> { + match input { + "." => Ok(EvaluationOperator::Dot), + ".." => Ok(EvaluationOperator::DotDot), _ => Err(()), } } diff --git a/crates/nu-parser/src/parse/parser.rs b/crates/nu-parser/src/parse/parser.rs index f63d505908..78ae113d53 100644 --- a/crates/nu-parser/src/parse/parser.rs +++ b/crates/nu-parser/src/parse/parser.rs @@ -14,6 +14,7 @@ use nom::sequence::*; use bigdecimal::BigDecimal; use derive_new::new; +use enumflags2::BitFlags; use log::trace; use nom::dbg; use nom::*; @@ -32,7 +33,7 @@ use serde::{Deserialize, Serialize}; use std::fmt::Debug; use std::str::FromStr; -macro_rules! operator { +macro_rules! cmp_operator { ($name:tt : $token:tt ) => { #[tracable_parser] pub fn $name(input: NomSpan) -> IResult { @@ -42,21 +43,38 @@ macro_rules! operator { Ok(( input, - TokenTreeBuilder::spanned_op(tag.fragment, Span::new(start, end)), + TokenTreeBuilder::spanned_cmp_op(tag.fragment, Span::new(start, end)), )) } }; } -operator! { gt: ">" } -operator! { lt: "<" } -operator! { gte: ">=" } -operator! { lte: "<=" } -operator! { eq: "==" } -operator! { neq: "!=" } -operator! { dot: "." } -operator! { cont: "=~" } -operator! { ncont: "!~" } +macro_rules! eval_operator { + ($name:tt : $token:tt ) => { + #[tracable_parser] + pub fn $name(input: NomSpan) -> IResult { + let start = input.offset; + let (input, tag) = tag($token)(input)?; + let end = input.offset; + + Ok(( + input, + TokenTreeBuilder::spanned_eval_op(tag.fragment, Span::new(start, end)), + )) + } + }; +} + +cmp_operator! { gt: ">" } +cmp_operator! { lt: "<" } +cmp_operator! { gte: ">=" } +cmp_operator! { lte: "<=" } +cmp_operator! { eq: "==" } +cmp_operator! { neq: "!=" } +cmp_operator! { cont: "=~" } +cmp_operator! { ncont: "!~" } +eval_operator! { dot: "." } +eval_operator! { dotdot: ".." } #[derive(Debug, Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize, Deserialize)] pub enum Number { @@ -213,6 +231,17 @@ pub fn raw_number(input: NomSpan) -> IResult { } } + let dotdot_result = dotdot(input); + + match dotdot_result { + // If we see a `..` immediately after an integer, it's a range, not a decimal + Ok((dotdot_input, _)) => { + return Ok((input, RawNumber::int(Span::new(start, input.offset)))) + } + + Err(_) => {} + } + let dot: IResult = tag(".")(input); let input = match dot { @@ -285,7 +314,7 @@ pub fn string(input: NomSpan) -> IResult { pub fn external(input: NomSpan) -> IResult { let start = input.offset; let (input, _) = tag("^")(input)?; - let (input, bare) = take_while(is_bare_char)(input)?; + let (input, bare) = take_while(is_file_char)(input)?; let end = input.offset; Ok(( @@ -294,52 +323,186 @@ pub fn external(input: NomSpan) -> IResult { )) } -#[tracable_parser] -pub fn pattern(input: NomSpan) -> IResult { - let start = input.offset; - let (input, _) = take_while1(is_start_glob_char)(input)?; - let (input, _) = take_while(is_glob_char)(input)?; +fn word<'a, T, U, V>( + start_predicate: impl Fn(NomSpan<'a>) -> IResult, U>, + next_predicate: impl Fn(NomSpan<'a>) -> IResult, V> + Copy, + into: impl Fn(Span) -> T, +) -> impl Fn(NomSpan<'a>) -> IResult, T> { + move |input: NomSpan| { + let start = input.offset; - let next_char = &input.fragment.chars().nth(0); + let (input, _) = start_predicate(input)?; + let (input, _) = many0(next_predicate)(input)?; - if let Some(next_char) = next_char { - if is_external_word_char(*next_char) { - return Err(nom::Err::Error(nom::error::make_error( - input, - nom::error::ErrorKind::TakeWhile1, - ))); + let next_char = &input.fragment.chars().nth(0); + + match next_char { + Some('.') => {} + Some(next_char) + if is_external_word_char(*next_char) || is_glob_specific_char(*next_char) => + { + return Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::TakeWhile1, + ))); + } + _ => {} } + + let end = input.offset; + + Ok((input, into(Span::new(start, end)))) } +} - let end = input.offset; - - Ok(( - input, - TokenTreeBuilder::spanned_pattern(Span::new(start, end)), - )) +pub fn matches(cond: fn(char) -> bool) -> impl Fn(NomSpan) -> IResult + Copy { + move |input: NomSpan| match input.iter_elements().next() { + Option::Some(c) if cond(c) => Ok((input.slice(1..), input.slice(0..1))), + _ => Err(nom::Err::Error(nom::error::ParseError::from_error_kind( + input, + nom::error::ErrorKind::Many0, + ))), + } } #[tracable_parser] -pub fn bare(input: NomSpan) -> IResult { - let start = input.offset; - let (input, _) = take_while1(is_start_bare_char)(input)?; - let (input, last) = take_while(is_bare_char)(input)?; +pub fn pattern(input: NomSpan) -> IResult { + word( + start_pattern, + matches(is_glob_char), + TokenTreeBuilder::spanned_pattern, + )(input) +} - let next_char = &input.fragment.chars().nth(0); - let prev_char = last.fragment.chars().nth(0); +#[tracable_parser] +pub fn start_pattern(input: NomSpan) -> IResult { + alt((take_while1(is_dot), matches(is_start_glob_char)))(input) +} - if let Some(next_char) = next_char { - if is_external_word_char(*next_char) || is_glob_specific_char(*next_char) { - return Err(nom::Err::Error(nom::error::make_error( - input, - nom::error::ErrorKind::TakeWhile1, - ))); +#[tracable_parser] +pub fn filename(input: NomSpan) -> IResult { + let start_pos = input.offset; + + let (mut input, mut saw_special) = match start_file_char(input) { + Err(err) => return Err(err), + Ok((input, special)) => (input, special), + }; + + loop { + if saw_special.is_empty() { + match continue_file_char(input) { + Err(_) => { + return Ok(( + input, + TokenTreeBuilder::spanned_bare((start_pos, input.offset)), + )) + } + Ok((next_input, special)) => { + saw_special |= special; + input = next_input; + } + } + } else { + let rest = after_sep_file(input); + + let (input, span, updated_special) = match rest { + Err(_) => (input, (start_pos, input.offset), saw_special), + Ok((input, new_special)) => { + (input, (start_pos, input.offset), saw_special | new_special) + } + }; + + if updated_special.contains(SawSpecial::Glob) { + return Ok((input, TokenTreeBuilder::spanned_pattern(span))); + } else { + return Ok((input, TokenTreeBuilder::spanned_bare(span))); + } } } +} - let end = input.offset; +#[derive(BitFlags, Copy, Clone, Eq, PartialEq)] +enum SawSpecial { + PathSeparator = 0b01, + Glob = 0b10, +} - Ok((input, TokenTreeBuilder::spanned_bare(Span::new(start, end)))) +#[tracable_parser] +fn start_file_char(input: NomSpan) -> IResult> { + let path_sep_result = special_file_char(input); + + match path_sep_result { + Ok((input, special)) => return Ok((input, special)), + + Err(_) => {} + } + + start_filename(input).map(|(input, output)| (input, BitFlags::empty())) +} + +#[tracable_parser] +fn continue_file_char(input: NomSpan) -> IResult> { + let path_sep_result = special_file_char(input); + + match path_sep_result { + Ok((input, special)) => return Ok((input, special)), + Err(_) => {} + } + + matches(is_file_char)(input).map(|(input, _)| (input, BitFlags::empty())) +} + +#[tracable_parser] +fn special_file_char(input: NomSpan) -> IResult> { + match matches(is_path_separator)(input) { + Ok((input, _)) => return Ok((input, BitFlags::empty() | SawSpecial::PathSeparator)), + Err(_) => {} + } + + let (input, _) = matches(is_glob_specific_char)(input)?; + + Ok((input, BitFlags::empty() | SawSpecial::Glob)) +} + +#[tracable_parser] +fn after_sep_file(input: NomSpan) -> IResult> { + fn after_sep_char(c: char) -> bool { + is_external_word_char(c) || is_file_char(c) || c == '.' + } + + let start = input.offset; + let original_input = input; + let mut input = input; + + let (input, after_glob) = take_while1(after_sep_char)(input)?; + + let slice = original_input.slice(0..input.offset - start); + + let saw_special = if slice.fragment.chars().any(is_glob_specific_char) { + BitFlags::empty() | SawSpecial::Glob + } else { + BitFlags::empty() + }; + + Ok((input, saw_special)) +} + +pub fn start_filename(input: NomSpan) -> IResult { + alt((take_while1(is_dot), matches(is_start_file_char)))(input) +} + +#[tracable_parser] +pub fn member(input: NomSpan) -> IResult { + word( + matches(is_start_member_char), + matches(is_member_char), + TokenTreeBuilder::spanned_bare, + )(input) +} + +#[tracable_parser] +pub fn ident(input: NomSpan) -> IResult { + word(matches(is_id_start), matches(is_id_continue), Tag::from)(input) } #[tracable_parser] @@ -348,10 +511,7 @@ pub fn external_word(input: NomSpan) -> IResult { let (input, _) = take_while1(is_external_word_char)(input)?; let end = input.offset; - Ok(( - input, - TokenTreeBuilder::spanned_external_word(Span::new(start, end)), - )) + Ok((input, TokenTreeBuilder::spanned_external_word((start, end)))) } #[tracable_parser] @@ -367,21 +527,40 @@ pub fn var(input: NomSpan) -> IResult { )) } -#[tracable_parser] -pub fn ident(input: NomSpan) -> IResult { - let start = input.offset; - let (input, _) = take_while1(is_start_bare_char)(input)?; - let (input, _) = take_while(is_bare_char)(input)?; - let end = input.offset; +fn tight<'a>( + parser: impl Fn(NomSpan<'a>) -> IResult, Vec>, +) -> impl Fn(NomSpan<'a>) -> IResult, Vec> { + move |input: NomSpan| { + let mut result = vec![]; + let (input, head) = parser(input)?; + result.extend(head); - Ok((input, Tag::from((start, end, None)))) + let (input, tail) = opt(alt((many1(range_continuation), many1(dot_member))))(input)?; + + let next_char = &input.fragment.chars().nth(0); + + if is_boundary(*next_char) { + if let Some(tail) = tail { + for tokens in tail { + result.extend(tokens); + } + } + + Ok((input, result)) + } else { + Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::Many0, + ))) + } + } } #[tracable_parser] pub fn flag(input: NomSpan) -> IResult { let start = input.offset; let (input, _) = tag("--")(input)?; - let (input, bare) = bare(input)?; + let (input, bare) = filename(input)?; let end = input.offset; Ok(( @@ -394,7 +573,7 @@ pub fn flag(input: NomSpan) -> IResult { pub fn shorthand(input: NomSpan) -> IResult { let start = input.offset; let (input, _) = tag("-")(input)?; - let (input, bare) = bare(input)?; + let (input, bare) = filename(input)?; let end = input.offset; Ok(( @@ -413,47 +592,97 @@ pub fn leaf(input: NomSpan) -> IResult { #[tracable_parser] pub fn token_list(input: NomSpan) -> IResult>> { let start = input.offset; - let (input, first) = node(input)?; + let mut node_list = vec![]; - let (input, mut list) = many0(pair(alt((whitespace, dot)), node))(input)?; + let mut next_input = input; + let mut before_space_input: Option = None; + let mut final_space_tokens = 0; - let end = input.offset; + loop { + let node_result = tight_node(next_input); - Ok(( - input, - make_token_list(first, list, None).spanned(Span::new(start, end)), - )) + let (after_node_input, next_nodes) = match node_result { + Err(_) => { + if let Some(before_space_input) = before_space_input { + next_input = before_space_input; + + for _ in 0..final_space_tokens { + node_list.pop(); + } + } + + break; + } + Ok((after_node_input, next_node)) => (after_node_input, next_node), + }; + + node_list.extend(next_nodes); + + // Special case that allows a parenthesized expression to immediate follow another + // token without a space, which could represent a type annotation. + let maybe_type = delimited_paren(after_node_input); + + let after_maybe_type_input = match maybe_type { + Err(_) => after_node_input, + Ok((after_maybe_type_input, parens)) => { + node_list.push(parens); + after_maybe_type_input + } + }; + + let maybe_space = any_space(after_maybe_type_input); + + let after_space_input = match maybe_space { + Err(_) => { + next_input = after_maybe_type_input; + + break; + } + Ok((after_space_input, space)) => { + final_space_tokens = space.len(); + node_list.extend(space); + before_space_input = Some(after_maybe_type_input); + after_space_input + } + }; + + next_input = after_space_input; + } + + let end = next_input.offset; + + Ok((next_input, node_list.spanned(Span::new(start, end)))) } #[tracable_parser] pub fn spaced_token_list(input: NomSpan) -> IResult>> { let start = input.offset; - let (input, pre_ws) = opt(whitespace)(input)?; + let (input, pre_ws) = opt(any_space)(input)?; let (input, items) = token_list(input)?; - let (input, post_ws) = opt(whitespace)(input)?; + let (input, post_ws) = opt(any_space)(input)?; let end = input.offset; let mut out = vec![]; - out.extend(pre_ws); + pre_ws.map(|pre_ws| out.extend(pre_ws)); out.extend(items.item); - out.extend(post_ws); + post_ws.map(|post_ws| out.extend(post_ws)); Ok((input, out.spanned(Span::new(start, end)))) } fn make_token_list( first: Vec, - list: Vec<(TokenNode, Vec)>, + list: Vec<(Vec, Vec)>, sp_right: Option, ) -> Vec { let mut nodes = vec![]; nodes.extend(first); - for (left, right) in list { - nodes.push(left); - nodes.extend(right); + for (sep, list) in list { + nodes.extend(sep); + nodes.extend(list); } if let Some(sp_right) = sp_right { @@ -463,6 +692,15 @@ fn make_token_list( nodes } +#[tracable_parser] +pub fn separator(input: NomSpan) -> IResult { + let left = input.offset; + let (input, ws1) = alt((tag(";"), tag("\n")))(input)?; + let right = input.offset; + + Ok((input, TokenTreeBuilder::spanned_sep(Span::new(left, right)))) +} + #[tracable_parser] pub fn whitespace(input: NomSpan) -> IResult { let left = input.offset; @@ -472,6 +710,30 @@ pub fn whitespace(input: NomSpan) -> IResult { Ok((input, TokenTreeBuilder::spanned_ws(Span::new(left, right)))) } +#[tracable_parser] +pub fn any_space(input: NomSpan) -> IResult> { + let left = input.offset; + let (input, tokens) = many1(alt((whitespace, separator, comment)))(input)?; + let right = input.offset; + + Ok((input, tokens)) +} + +#[tracable_parser] +pub fn comment(input: NomSpan) -> IResult { + let left = input.offset; + let (input, start) = tag("#")(input)?; + let (input, rest) = not_line_ending(input)?; + let right = input.offset; + + let span = (start.offset + 1, right); + + Ok(( + input, + TokenTreeBuilder::spanned_comment(span, Span::new(left, right)), + )) +} + pub fn delimited( input: NomSpan, delimiter: Delimiter, @@ -541,62 +803,43 @@ pub fn raw_call(input: NomSpan) -> IResult> { } #[tracable_parser] -pub fn bare_path(input: NomSpan) -> IResult> { - let (input, head) = alt((bare, dot))(input)?; +pub fn range_continuation(input: NomSpan) -> IResult> { + let original = input; - let (input, tail) = many0(alt((bare, dot, string)))(input)?; + let mut result = vec![]; - let next_char = &input.fragment.chars().nth(0); + let (input, dotdot_result) = dotdot(input)?; + result.push(dotdot_result); + let (input, node_result) = tight_node(input)?; + result.extend(node_result); - if is_boundary(*next_char) { - let mut result = vec![head]; - result.extend(tail); - - Ok((input, result)) - } else { - Err(nom::Err::Error(nom::error::make_error( - input, - nom::error::ErrorKind::Many0, - ))) - } + Ok((input, result)) } #[tracable_parser] -pub fn pattern_path(input: NomSpan) -> IResult> { - let (input, head) = alt((pattern, dot))(input)?; +pub fn dot_member(input: NomSpan) -> IResult> { + let (input, dot_result) = dot(input)?; + let (input, member_result) = any_member(input)?; - let (input, tail) = many0(alt((pattern, dot, string)))(input)?; - - let next_char = &input.fragment.chars().nth(0); - - if is_boundary(*next_char) { - let mut result = vec![head]; - result.extend(tail); - - Ok((input, result)) - } else { - Err(nom::Err::Error(nom::error::make_error( - input, - nom::error::ErrorKind::Many0, - ))) - } + Ok((input, vec![dot_result, member_result])) } #[tracable_parser] -pub fn node1(input: NomSpan) -> IResult { - alt((leaf, bare, pattern, external_word, delimited_paren))(input) +pub fn any_member(input: NomSpan) -> IResult { + alt((number, string, member))(input) } #[tracable_parser] -pub fn node(input: NomSpan) -> IResult> { +pub fn tight_node(input: NomSpan) -> IResult> { alt(( - to_list(leaf), - bare_path, - pattern_path, + tight(to_list(leaf)), + tight(to_list(filename)), + tight(to_list(pattern)), + to_list(comment), to_list(external_word), - to_list(delimited_paren), - to_list(delimited_brace), - to_list(delimited_square), + tight(to_list(delimited_paren)), + tight(to_list(delimited_brace)), + tight(to_list(delimited_square)), ))(input) } @@ -649,6 +892,23 @@ pub fn pipeline(input: NomSpan) -> IResult { )) } +#[tracable_parser] +pub fn module(input: NomSpan) -> IResult { + let (input, tokens) = spaced_token_list(input)?; + + if input.input_len() != 0 { + return Err(Err::Error(error_position!( + input, + nom::error::ErrorKind::Eof + ))); + } + + Ok(( + input, + TokenTreeBuilder::spanned_token_list(tokens.item, tokens.span), + )) +} + fn parse_int(frag: &str, neg: Option) -> i64 { let int = FromStr::from_str(frag).unwrap(); @@ -661,7 +921,7 @@ fn parse_int(frag: &str, neg: Option) -> i64 { fn is_boundary(c: Option) -> bool { match c { None => true, - Some(')') | Some(']') | Some('}') => true, + Some(')') | Some(']') | Some('}') | Some('(') => true, Some(c) if c.is_whitespace() => true, _ => false, } @@ -682,14 +942,25 @@ fn is_glob_specific_char(c: char) -> bool { } fn is_start_glob_char(c: char) -> bool { - is_start_bare_char(c) || is_glob_specific_char(c) || c == '.' + is_start_file_char(c) || is_glob_specific_char(c) || c == '.' } fn is_glob_char(c: char) -> bool { - is_bare_char(c) || is_glob_specific_char(c) + is_file_char(c) || is_glob_specific_char(c) } -fn is_start_bare_char(c: char) -> bool { +fn is_dot(c: char) -> bool { + c == '.' +} + +fn is_path_separator(c: char) -> bool { + match c { + '\\' | '/' | ':' => true, + _ => false, + } +} + +fn is_start_file_char(c: char) -> bool { match c { '+' => false, _ if c.is_alphanumeric() => true, @@ -698,11 +969,12 @@ fn is_start_bare_char(c: char) -> bool { '_' => true, '-' => true, '~' => true, + '.' => true, _ => false, } } -fn is_bare_char(c: char) -> bool { +fn is_file_char(c: char) -> bool { match c { '+' => true, _ if c.is_alphanumeric() => true, @@ -718,6 +990,24 @@ fn is_bare_char(c: char) -> bool { } } +fn is_start_member_char(c: char) -> bool { + match c { + _ if c.is_alphanumeric() => true, + '_' => true, + '-' => true, + _ => false, + } +} + +fn is_member_char(c: char) -> bool { + match c { + _ if c.is_alphanumeric() => true, + '_' => true, + '-' => true, + _ => false, + } +} + fn is_id_start(c: char) -> bool { unicode_xid::UnicodeXID::is_xid_start(c) } @@ -775,6 +1065,7 @@ mod tests { (<$parser:tt> $source:tt -> $tokens:expr) => { let result = apply($parser, stringify!($parser), $source); + let (expected_tree, expected_source) = TokenTreeBuilder::build($tokens); if result != expected_tree { @@ -884,22 +1175,36 @@ mod tests { fn test_simple_path() { equal_tokens! { - "chrome.exe" -> b::token_list(vec![b::bare("chrome"), b::op(Operator::Dot), b::bare("exe")]) + "chrome.exe" -> b::token_list(vec![b::bare("chrome"), b::dot(), b::bare("exe")]) } equal_tokens! { - ".azure" -> b::token_list(vec![b::op(Operator::Dot), b::bare("azure")]) + ".azure" -> b::token_list(vec![b::bare(".azure")]) } equal_tokens! { - r"C:\windows\system.dll" -> b::token_list(vec![b::bare(r"C:\windows\system"), b::op(Operator::Dot), b::bare("dll")]) + r"C:\windows\system.dll" -> b::token_list(vec![b::bare(r"C:\windows\system.dll")]) } equal_tokens! { - r"C:\Code\-testing\my_tests.js" -> b::token_list(vec![b::bare(r"C:\Code\-testing\my_tests"), b::op(Operator::Dot), b::bare("js")]) + r"C:\Code\-testing\my_tests.js" -> b::token_list(vec![b::bare(r"C:\Code\-testing\my_tests.js")]) + } + + equal_tokens! { + + r"C:\Users\example\AppData\Local\Temp\.tmpZ4TVQ2\cd_test_8" -> b::token_list(vec![b::bare(r"C:\Users\example\AppData\Local\Temp\.tmpZ4TVQ2\cd_test_8")]) + } + + equal_tokens! { + + r"cd C:\Users\wycat\AppData\Local\Temp\.tmpaj5JKi\cd_test_11" -> b::pipeline(vec![vec![ + b::bare("cd"), + b::sp(), + b::bare(r"C:\Users\wycat\AppData\Local\Temp\.tmpaj5JKi\cd_test_11") + ]]) } } @@ -949,7 +1254,7 @@ mod tests { fn test_dot_prefixed_name() { equal_tokens! { - ".azure" -> b::token_list(vec![b::op("."), b::bare("azure")]) + ".azure" -> b::token_list(vec![b::bare(".azure")]) } } @@ -1003,33 +1308,43 @@ mod tests { } } + #[test] + fn test_range() { + let _ = pretty_env_logger::try_init(); + + equal_tokens! { + + "0..2" -> b::token_list(vec![b::int(0), b::dotdot(), b::int(2)]) + } + } + #[test] fn test_path() { let _ = pretty_env_logger::try_init(); equal_tokens! { - "$it.print" -> b::token_list(vec![b::var("it"), b::op("."), b::bare("print")]) + "$it.print" -> b::token_list(vec![b::var("it"), b::dot(), b::bare("print")]) } equal_tokens! { - "$it.0" -> b::token_list(vec![b::var("it"), b::op("."), b::int(0)]) + "$it.0" -> b::token_list(vec![b::var("it"), b::dot(), b::int(0)]) } equal_tokens! { - "$head.part1.part2" -> b::token_list(vec![b::var("head"), b::op("."), b::bare("part1"), b::op("."), b::bare("part2")]) + "$head.part1.part2" -> b::token_list(vec![b::var("head"), b::dot(), b::bare("part1"), b::dot(), b::bare("part2")]) } equal_tokens! { - "( hello ).world" -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::op("."), b::bare("world")]) + "( hello ).world" -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::dot(), b::bare("world")]) } equal_tokens! { - r#"( hello )."world""# -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::op("."), b::string("world")]) + r#"( hello )."world""# -> b::token_list(vec![b::parens(vec![b::sp(), b::bare("hello"), b::sp()]), b::dot(), b::string("world")]) } } @@ -1042,11 +1357,11 @@ mod tests { b::parens(vec![ b::sp(), b::var("it"), - b::op("."), + b::dot(), b::bare("is"), - b::op("."), + b::dot(), b::string("great news"), - b::op("."), + b::dot(), b::bare("right"), b::sp(), b::bare("yep"), @@ -1054,7 +1369,7 @@ mod tests { b::var("yep"), b::sp() ]), - b::op("."), b::string("world")] + b::dot(), b::string("world")] ) } @@ -1063,9 +1378,9 @@ mod tests { r#"$it."are PAS".0"# -> b::token_list( vec![ b::var("it"), - b::op("."), + b::dot(), b::string("are PAS"), - b::op("."), + b::dot(), b::int(0), ] ) @@ -1076,17 +1391,17 @@ mod tests { fn test_smoke_single_command() { equal_tokens! { - "git add ." -> b::token_list(vec![b::bare("git"), b::sp(), b::bare("add"), b::sp(), b::op(".")]) + "git add ." -> b::token_list(vec![b::bare("git"), b::sp(), b::bare("add"), b::sp(), b::bare(".")]) } equal_tokens! { - "open Cargo.toml" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::op("."), b::bare("toml")]) + "open Cargo.toml" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::dot(), b::bare("toml")]) } equal_tokens! { - "select package.version" -> b::token_list(vec![b::bare("select"), b::sp(), b::bare("package"), b::op("."), b::bare("version")]) + "select package.version" -> b::token_list(vec![b::bare("select"), b::sp(), b::bare("package"), b::dot(), b::bare("version")]) } equal_tokens! { @@ -1096,12 +1411,12 @@ mod tests { equal_tokens! { - "open Cargo.toml --raw" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::op("."), b::bare("toml"), b::sp(), b::flag("raw")]) + "open Cargo.toml --raw" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::dot(), b::bare("toml"), b::sp(), b::flag("raw")]) } equal_tokens! { - "open Cargo.toml -r" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::op("."), b::bare("toml"), b::sp(), b::shorthand("r")]) + "open Cargo.toml -r" -> b::token_list(vec![b::bare("open"), b::sp(), b::bare("Cargo"), b::dot(), b::bare("toml"), b::sp(), b::shorthand("r")]) } equal_tokens! { @@ -1117,7 +1432,7 @@ mod tests { b::sp(), b::flag("patch"), b::sp(), - b::bare("package"), b::op("."), b::bare("version") + b::bare("package"), b::dot(), b::bare("version") ] ) } @@ -1197,7 +1512,7 @@ mod tests { b::bare("where"), b::sp(), b::bare("cpu"), - b::op("."), + b::dot(), b::string("max ghz"), b::sp(), b::op(">"), @@ -1207,6 +1522,34 @@ mod tests { ); } + #[test] + fn test_signature() { + let _ = pretty_env_logger::try_init(); + + equal_tokens!( + + "def cd\n # Change to a new path.\n optional directory(Path) # the directory to change to\nend" -> + b::token_list(vec![ + b::bare("def"), + b::sp(), + b::bare("cd"), + b::sep("\n"), + b::ws(" "), + b::comment(" Change to a new path."), + b::sep("\n"), + b::ws(" "), + b::bare("optional"), + b::sp(), + b::bare("directory"), + b::parens(vec![b::bare("Path")]), + b::sp(), + b::comment(" the directory to change to"), + b::sep("\n"), + b::bare("end") + ]) + ); + } + // #[test] // fn test_smoke_pipeline() { // let _ = pretty_env_logger::try_init(); @@ -1279,7 +1622,18 @@ mod tests { desc: &str, string: &str, ) -> TokenNode { - f(nom_input(string)).unwrap().1 + let result = f(nom_input(string)); + + match result { + Ok(value) => value.1, + Err(err) => { + let err = nu_errors::ShellError::parse_error(err); + + println!("{:?}", string); + crate::hir::baseline_parse::tests::print_err(err, &nu_source::Text::from(string)); + panic!("test failed") + } + } } fn span((left, right): (usize, usize)) -> Span { diff --git a/crates/nu-parser/src/parse/token_tree.rs b/crates/nu-parser/src/parse/token_tree.rs index 2869a4b7ae..012c548b1b 100644 --- a/crates/nu-parser/src/parse/token_tree.rs +++ b/crates/nu-parser/src/parse/token_tree.rs @@ -1,4 +1,4 @@ -use crate::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*}; +use crate::parse::{call_node::*, comment::*, flag::*, operator::*, pipeline::*, tokens::*}; use derive_new::new; use getset::Getters; use nu_errors::{ParseError, ShellError}; @@ -18,7 +18,9 @@ pub enum TokenNode { Delimited(Spanned), Pipeline(Pipeline), Flag(Flag), + Comment(Comment), Whitespace(Span), + Separator(Span), Error(Spanned), } @@ -39,14 +41,32 @@ impl PrettyDebugWithSource for TokenNode { "whitespace", b::description(format!("{:?}", space.slice(source))), ), + TokenNode::Separator(span) => b::typed( + "separator", + b::description(format!("{:?}", span.slice(source))), + ), + TokenNode::Comment(comment) => { + b::typed("comment", b::description(comment.text.slice(source))) + } TokenNode::Error(_) => b::error("error"), } } } -impl HasSpan for TokenNode { - fn span(&self) -> Span { - self.get_span() +impl ShellTypeName for TokenNode { + fn type_name(&self) -> &'static str { + match self { + TokenNode::Token(t) => t.type_name(), + TokenNode::Nodes(_) => "nodes", + TokenNode::Call(_) => "command", + TokenNode::Delimited(d) => d.type_name(), + TokenNode::Pipeline(_) => "pipeline", + TokenNode::Flag(_) => "flag", + TokenNode::Whitespace(_) => "whitespace", + TokenNode::Separator(_) => "separator", + TokenNode::Comment(_) => "comment", + TokenNode::Error(_) => "error", + } } } @@ -107,12 +127,12 @@ impl fmt::Debug for DebugTokenNode<'_> { impl From<&TokenNode> for Span { fn from(token: &TokenNode) -> Span { - token.get_span() + token.span() } } -impl TokenNode { - pub fn get_span(&self) -> Span { +impl HasSpan for TokenNode { + fn span(&self) -> Span { match self { TokenNode::Token(t) => t.span, TokenNode::Nodes(t) => t.span, @@ -121,27 +141,14 @@ impl TokenNode { TokenNode::Pipeline(s) => s.span, TokenNode::Flag(s) => s.span, TokenNode::Whitespace(s) => *s, + TokenNode::Separator(s) => *s, + TokenNode::Comment(c) => c.span(), TokenNode::Error(s) => s.span, } } +} - pub fn type_name(&self) -> &'static str { - match self { - TokenNode::Token(t) => t.type_name(), - TokenNode::Nodes(_) => "nodes", - TokenNode::Call(_) => "command", - TokenNode::Delimited(d) => d.type_name(), - TokenNode::Pipeline(_) => "pipeline", - TokenNode::Flag(_) => "flag", - TokenNode::Whitespace(_) => "whitespace", - TokenNode::Error(_) => "error", - } - } - - pub fn spanned_type_name(&self) -> Spanned<&'static str> { - self.type_name().spanned(self.span()) - } - +impl TokenNode { pub fn tagged_type_name(&self) -> Tagged<&'static str> { self.type_name().tagged(self.span()) } @@ -244,7 +251,7 @@ impl TokenNode { pub fn is_dot(&self) -> bool { match self { TokenNode::Token(Token { - unspanned: UnspannedToken::Operator(Operator::Dot), + unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::Dot), .. }) => true, _ => false, @@ -421,7 +428,7 @@ impl TokenNode { pub fn expect_dot(&self) -> Span { match self { TokenNode::Token(Token { - unspanned: UnspannedToken::Operator(Operator::Dot), + unspanned: UnspannedToken::EvaluationOperator(EvaluationOperator::Dot), span, }) => *span, other => panic!("Expected dot, found {:?}", other), diff --git a/crates/nu-parser/src/parse/token_tree_builder.rs b/crates/nu-parser/src/parse/token_tree_builder.rs index 7b82cb4131..f88cb8b8e6 100644 --- a/crates/nu-parser/src/parse/token_tree_builder.rs +++ b/crates/nu-parser/src/parse/token_tree_builder.rs @@ -1,6 +1,7 @@ use crate::parse::call_node::CallNode; +use crate::parse::comment::Comment; use crate::parse::flag::{Flag, FlagKind}; -use crate::parse::operator::Operator; +use crate::parse::operator::{CompareOperator, EvaluationOperator}; use crate::parse::pipeline::{Pipeline, PipelineElement}; use crate::parse::token_tree::{DelimitedNode, Delimiter, TokenNode}; use crate::parse::tokens::{RawNumber, UnspannedToken}; @@ -96,7 +97,7 @@ impl TokenTreeBuilder { TokenNode::Nodes(input.spanned(span.into())) } - pub fn op(input: impl Into) -> CurriedToken { + pub fn op(input: impl Into) -> CurriedToken { let input = input.into(); Box::new(move |b| { @@ -104,12 +105,39 @@ impl TokenTreeBuilder { b.pos = end; - TokenTreeBuilder::spanned_op(input, Span::new(start, end)) + TokenTreeBuilder::spanned_cmp_op(input, Span::new(start, end)) }) } - pub fn spanned_op(input: impl Into, span: impl Into) -> TokenNode { - TokenNode::Token(UnspannedToken::Operator(input.into()).into_token(span)) + pub fn spanned_cmp_op(input: impl Into, span: impl Into) -> TokenNode { + TokenNode::Token(UnspannedToken::CompareOperator(input.into()).into_token(span)) + } + + pub fn dot() -> CurriedToken { + Box::new(move |b| { + let (start, end) = b.consume("."); + + b.pos = end; + + TokenTreeBuilder::spanned_eval_op(".", Span::new(start, end)) + }) + } + + pub fn dotdot() -> CurriedToken { + Box::new(move |b| { + let (start, end) = b.consume(".."); + + b.pos = end; + + TokenTreeBuilder::spanned_eval_op("..", Span::new(start, end)) + }) + } + + pub fn spanned_eval_op( + input: impl Into, + span: impl Into, + ) -> TokenNode { + TokenNode::Token(UnspannedToken::EvaluationOperator(input.into()).into_token(span)) } pub fn string(input: impl Into) -> CurriedToken { @@ -398,6 +426,36 @@ impl TokenTreeBuilder { TokenNode::Whitespace(span.into()) } + pub fn sep(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (start, end) = b.consume(&input); + TokenTreeBuilder::spanned_sep(Span::new(start, end)) + }) + } + + pub fn spanned_sep(span: impl Into) -> TokenNode { + TokenNode::Separator(span.into()) + } + + pub fn comment(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let outer_start = b.pos; + b.consume("#"); + let (start, end) = b.consume(&input); + let outer_end = b.pos; + + TokenTreeBuilder::spanned_comment((start, end), (outer_start, outer_end)) + }) + } + + pub fn spanned_comment(input: impl Into, span: impl Into) -> TokenNode { + TokenNode::Comment(Comment::line(input, span)) + } + fn consume(&mut self, input: &str) -> (usize, usize) { let start = self.pos; self.pos += input.len(); diff --git a/crates/nu-parser/src/parse/tokens.rs b/crates/nu-parser/src/parse/tokens.rs index 8430725313..4cb958c957 100644 --- a/crates/nu-parser/src/parse/tokens.rs +++ b/crates/nu-parser/src/parse/tokens.rs @@ -1,5 +1,5 @@ use crate::parse::parser::Number; -use crate::Operator; +use crate::{CompareOperator, EvaluationOperator}; use bigdecimal::BigDecimal; use nu_protocol::ShellTypeName; use nu_source::{ @@ -13,7 +13,8 @@ use std::str::FromStr; #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum UnspannedToken { Number(RawNumber), - Operator(Operator), + CompareOperator(CompareOperator), + EvaluationOperator(EvaluationOperator), String(Span), Variable(Span), ExternalCommand(Span), @@ -35,7 +36,9 @@ impl ShellTypeName for UnspannedToken { fn type_name(&self) -> &'static str { match self { UnspannedToken::Number(_) => "number", - UnspannedToken::Operator(..) => "operator", + UnspannedToken::CompareOperator(..) => "comparison operator", + UnspannedToken::EvaluationOperator(EvaluationOperator::Dot) => "dot", + UnspannedToken::EvaluationOperator(EvaluationOperator::DotDot) => "dotdot", UnspannedToken::String(_) => "string", UnspannedToken::Variable(_) => "variable", UnspannedToken::ExternalCommand(_) => "syntax error", @@ -111,7 +114,8 @@ impl PrettyDebugWithSource for Token { fn pretty_debug(&self, source: &str) -> DebugDocBuilder { match self.unspanned { UnspannedToken::Number(number) => number.pretty_debug(source), - UnspannedToken::Operator(operator) => operator.pretty(), + UnspannedToken::CompareOperator(operator) => operator.pretty(), + UnspannedToken::EvaluationOperator(operator) => operator.pretty(), UnspannedToken::String(_) => b::primitive(self.span.slice(source)), UnspannedToken::Variable(_) => b::var(self.span.slice(source)), UnspannedToken::ExternalCommand(_) => b::primitive(self.span.slice(source)), @@ -155,9 +159,9 @@ impl Token { } } - pub fn extract_operator(&self) -> Option> { + pub fn extract_operator(&self) -> Option> { match self.unspanned { - UnspannedToken::Operator(operator) => Some(operator.spanned(self.span)), + UnspannedToken::CompareOperator(operator) => Some(operator.spanned(self.span)), _ => None, } } diff --git a/crates/nu-protocol/src/lib.rs b/crates/nu-protocol/src/lib.rs index f97fed9139..f77dc6f85a 100644 --- a/crates/nu-protocol/src/lib.rs +++ b/crates/nu-protocol/src/lib.rs @@ -8,6 +8,7 @@ mod return_value; mod signature; mod syntax_shape; mod type_name; +mod type_shape; mod value; pub use crate::call_info::{CallInfo, EvaluatedArgs}; @@ -17,9 +18,11 @@ pub use crate::return_value::{CommandAction, ReturnSuccess, ReturnValue}; pub use crate::signature::{NamedType, PositionalType, Signature}; pub use crate::syntax_shape::SyntaxShape; pub use crate::type_name::{PrettyType, ShellTypeName, SpannedTypeName}; +pub use crate::type_shape::{Row as RowType, Type}; pub use crate::value::column_path::{did_you_mean, ColumnPath, PathMember, UnspannedPathMember}; pub use crate::value::dict::{Dictionary, TaggedDictBuilder}; pub use crate::value::evaluate::{Evaluate, EvaluateTrait, Scope}; pub use crate::value::primitive::format_primitive; pub use crate::value::primitive::Primitive; +pub use crate::value::range::{Range, RangeInclusion}; pub use crate::value::{UntaggedValue, Value}; diff --git a/crates/nu-protocol/src/signature.rs b/crates/nu-protocol/src/signature.rs index 7919506131..b75f6a9d4c 100644 --- a/crates/nu-protocol/src/signature.rs +++ b/crates/nu-protocol/src/signature.rs @@ -1,4 +1,5 @@ use crate::syntax_shape::SyntaxShape; +use crate::type_shape::Type; use indexmap::IndexMap; use nu_source::{b, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource}; use serde::{Deserialize, Serialize}; @@ -76,6 +77,8 @@ pub struct Signature { pub positional: Vec<(PositionalType, Description)>, pub rest_positional: Option<(SyntaxShape, Description)>, pub named: IndexMap, + pub yields: Option, + pub input: Option, pub is_filter: bool, } @@ -98,14 +101,16 @@ impl PrettyDebugWithSource for Signature { } impl Signature { - pub fn new(name: String) -> Signature { + pub fn new(name: impl Into) -> Signature { Signature { - name, + name: name.into(), usage: String::new(), positional: vec![], rest_positional: None, named: IndexMap::new(), is_filter: false, + yields: None, + input: None, } } @@ -186,4 +191,14 @@ impl Signature { self.rest_positional = Some((ty, desc.into())); self } + + pub fn yields(mut self, ty: Type) -> Signature { + self.yields = Some(ty); + self + } + + pub fn input(mut self, ty: Type) -> Signature { + self.input = Some(ty); + self + } } diff --git a/crates/nu-protocol/src/syntax_shape.rs b/crates/nu-protocol/src/syntax_shape.rs index 7f6d486bfd..3f1d50f8b9 100644 --- a/crates/nu-protocol/src/syntax_shape.rs +++ b/crates/nu-protocol/src/syntax_shape.rs @@ -8,6 +8,7 @@ pub enum SyntaxShape { Member, ColumnPath, Number, + Range, Int, Path, Pattern, @@ -22,6 +23,7 @@ impl PrettyDebug for SyntaxShape { SyntaxShape::Member => "member shape", SyntaxShape::ColumnPath => "column path shape", SyntaxShape::Number => "number shape", + SyntaxShape::Range => "range shape", SyntaxShape::Int => "integer shape", SyntaxShape::Path => "file path shape", SyntaxShape::Pattern => "pattern shape", diff --git a/crates/nu-protocol/src/type_shape.rs b/crates/nu-protocol/src/type_shape.rs new file mode 100644 index 0000000000..be4aaaaabe --- /dev/null +++ b/crates/nu-protocol/src/type_shape.rs @@ -0,0 +1,382 @@ +use crate::value::dict::Dictionary; +use crate::value::primitive::Primitive; +use crate::value::range::RangeInclusion; +use crate::value::{UntaggedValue, Value}; +use derive_new::new; +use nu_source::{b, DebugDoc, DebugDocBuilder, PrettyDebug}; +use serde::{Deserialize, Deserializer, Serialize}; +use std::collections::BTreeMap; +use std::fmt::Debug; +use std::hash::Hash; + +/** + This file describes the structural types of the nushell system. + + Its primary purpose today is to identify "equivalent" values for the purpose + of merging rows into a single table or identify rows in a table that have the + same shape for reflection. +*/ + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, new)] +pub struct RangeType { + from: (Type, RangeInclusion), + to: (Type, RangeInclusion), +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +pub enum Type { + Nothing, + Int, + Range(Box), + Decimal, + Bytesize, + String, + Line, + ColumnPath, + Pattern, + Boolean, + Date, + Duration, + Path, + Binary, + + Row(Row), + Table(Vec), + + // TODO: Block arguments + Block, + // TODO: Error type + Error, + + // Stream markers (used as bookend markers rather than actual values) + BeginningOfStream, + EndOfStream, +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, new)] +pub struct Row { + #[new(default)] + map: BTreeMap, +} + +impl Serialize for Row { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_map(self.map.iter()) + } +} + +impl<'de> Deserialize<'de> for Row { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct RowVisitor; + + impl<'de> serde::de::Visitor<'de> for RowVisitor { + type Value = Row; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "a row") + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let mut new_map = BTreeMap::new(); + + loop { + let entry = map.next_entry()?; + + match entry { + None => return Ok(Row { map: new_map }), + Some((key, value)) => { + new_map.insert(key, value); + } + } + } + } + } + deserializer.deserialize_map(RowVisitor) + } +} + +impl Type { + pub fn from_primitive(primitive: &Primitive) -> Type { + match primitive { + Primitive::Nothing => Type::Nothing, + Primitive::Int(_) => Type::Int, + Primitive::Range(range) => { + let (left_value, left_inclusion) = &range.from; + let (right_value, right_inclusion) = &range.to; + + let left_type = (Type::from_primitive(left_value), *left_inclusion); + let right_type = (Type::from_primitive(right_value), *right_inclusion); + + let range = RangeType::new(left_type, right_type); + Type::Range(Box::new(range)) + } + Primitive::Decimal(_) => Type::Decimal, + Primitive::Bytes(_) => Type::Bytesize, + Primitive::String(_) => Type::String, + Primitive::Line(_) => Type::Line, + Primitive::ColumnPath(_) => Type::ColumnPath, + Primitive::Pattern(_) => Type::Pattern, + Primitive::Boolean(_) => Type::Boolean, + Primitive::Date(_) => Type::Date, + Primitive::Duration(_) => Type::Duration, + Primitive::Path(_) => Type::Path, + Primitive::Binary(_) => Type::Binary, + Primitive::BeginningOfStream => Type::BeginningOfStream, + Primitive::EndOfStream => Type::EndOfStream, + } + } + + pub fn from_dictionary(dictionary: &Dictionary) -> Type { + let mut map = BTreeMap::new(); + + for (key, value) in dictionary.entries.iter() { + let column = Column::String(key.clone()); + map.insert(column, Type::from_value(value)); + } + + Type::Row(Row { map }) + } + + pub fn from_table<'a>(table: impl IntoIterator) -> Type { + let mut vec = vec![]; + + for item in table.into_iter() { + vec.push(Type::from_value(item)) + } + + Type::Table(vec) + } + + pub fn from_value<'a>(value: impl Into<&'a UntaggedValue>) -> Type { + match value.into() { + UntaggedValue::Primitive(p) => Type::from_primitive(p), + UntaggedValue::Row(row) => Type::from_dictionary(row), + UntaggedValue::Table(table) => Type::from_table(table.iter()), + UntaggedValue::Error(_) => Type::Error, + UntaggedValue::Block(_) => Type::Block, + } + } +} + +impl PrettyDebug for Type { + fn pretty(&self) -> DebugDocBuilder { + match self { + Type::Nothing => ty("nothing"), + Type::Int => ty("integer"), + Type::Range(range) => { + let (left, left_inclusion) = &range.from; + let (right, right_inclusion) = &range.to; + + let left_bracket = b::delimiter(match left_inclusion { + RangeInclusion::Exclusive => "(", + RangeInclusion::Inclusive => "[", + }); + + let right_bracket = b::delimiter(match right_inclusion { + RangeInclusion::Exclusive => ")", + RangeInclusion::Inclusive => "]", + }); + + b::typed( + "range", + (left_bracket + + left.pretty() + + b::operator(",") + + b::space() + + right.pretty() + + right_bracket) + .group(), + ) + } + Type::Decimal => ty("decimal"), + Type::Bytesize => ty("bytesize"), + Type::String => ty("string"), + Type::Line => ty("line"), + Type::ColumnPath => ty("column-path"), + Type::Pattern => ty("pattern"), + Type::Boolean => ty("boolean"), + Type::Date => ty("date"), + Type::Duration => ty("duration"), + Type::Path => ty("path"), + Type::Binary => ty("binary"), + Type::Error => b::error("error"), + Type::BeginningOfStream => b::keyword("beginning-of-stream"), + Type::EndOfStream => b::keyword("end-of-stream"), + Type::Row(row) => (b::kind("row") + + b::space() + + b::intersperse( + row.map.iter().map(|(key, ty)| { + (b::key(match key { + Column::String(string) => string.clone(), + Column::Value => "".to_string(), + }) + b::delimit("(", ty.pretty(), ")").into_kind()) + .nest() + }), + b::space(), + ) + .nest()) + .nest(), + + Type::Table(table) => { + let mut group: Group> = Group::new(); + + for (i, item) in table.iter().enumerate() { + group.add(item.to_doc(), i); + } + + (b::kind("table") + b::space() + b::keyword("of")).group() + + b::space() + + (if group.len() == 1 { + let (doc, _) = group.into_iter().nth(0).unwrap(); + DebugDocBuilder::from_doc(doc) + } else { + b::intersperse( + group.into_iter().map(|(doc, rows)| { + (b::intersperse( + rows.iter().map(|(from, to)| { + if from == to { + b::description(from) + } else { + (b::description(from) + + b::space() + + b::keyword("to") + + b::space() + + b::description(to)) + .group() + } + }), + b::description(", "), + ) + b::description(":") + + b::space() + + DebugDocBuilder::from_doc(doc)) + .nest() + }), + b::space(), + ) + }) + } + Type::Block => ty("block"), + } + } +} + +#[derive(Debug, new)] +struct DebugEntry<'a> { + key: &'a Column, + value: &'a Type, +} + +impl<'a> PrettyDebug for DebugEntry<'a> { + fn pretty(&self) -> DebugDocBuilder { + (b::key(match self.key { + Column::String(string) => string.clone(), + Column::Value => format!(""), + }) + b::delimit("(", self.value.pretty(), ")").into_kind()) + } +} + +fn ty(name: impl std::fmt::Display) -> DebugDocBuilder { + b::kind(format!("{}", name)) +} + +pub trait GroupedValue: Debug + Clone { + type Item; + + fn new() -> Self; + fn merge(&mut self, value: Self::Item); +} + +impl GroupedValue for Vec<(usize, usize)> { + type Item = usize; + + fn new() -> Vec<(usize, usize)> { + vec![] + } + + fn merge(&mut self, new_value: usize) { + match self.last_mut() { + Some(value) if value.1 == new_value - 1 => { + value.1 += 1; + } + + _ => self.push((new_value, new_value)), + } + } +} + +#[derive(Debug)] +pub struct Group { + values: indexmap::IndexMap, +} + +impl Group +where + K: Debug + Eq + Hash, + G: GroupedValue, +{ + pub fn new() -> Group { + Group { + values: indexmap::IndexMap::default(), + } + } + + pub fn len(&self) -> usize { + self.values.len() + } + + pub fn into_iter(self) -> impl Iterator { + self.values.into_iter() + } + + pub fn add(&mut self, key: impl Into, value: impl Into) { + let key = key.into(); + let value = value.into(); + + let group = self.values.get_mut(&key); + + match group { + None => { + self.values.insert(key, { + let mut group = G::new(); + group.merge(value.into()); + group + }); + } + Some(group) => { + group.merge(value.into()); + } + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)] +pub enum Column { + String(String), + Value, +} + +impl Into for String { + fn into(self) -> Column { + Column::String(self) + } +} + +impl Into for &String { + fn into(self) -> Column { + Column::String(self.clone()) + } +} + +impl Into for &str { + fn into(self) -> Column { + Column::String(self.to_string()) + } +} diff --git a/crates/nu-protocol/src/value.rs b/crates/nu-protocol/src/value.rs index c5c6c47368..5082799996 100644 --- a/crates/nu-protocol/src/value.rs +++ b/crates/nu-protocol/src/value.rs @@ -4,6 +4,7 @@ mod debug; pub mod dict; pub mod evaluate; pub mod primitive; +pub mod range; mod serde_bigdecimal; mod serde_bigint; @@ -11,11 +12,12 @@ use crate::type_name::{ShellTypeName, SpannedTypeName}; use crate::value::dict::Dictionary; use crate::value::evaluate::Evaluate; use crate::value::primitive::Primitive; +use crate::value::range::{Range, RangeInclusion}; use crate::{ColumnPath, PathMember}; use bigdecimal::BigDecimal; use indexmap::IndexMap; use nu_errors::ShellError; -use nu_source::{AnchorLocation, HasSpan, Span, Tag}; +use nu_source::{AnchorLocation, HasSpan, Span, Spanned, Tag}; use num_bigint::BigInt; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -156,6 +158,13 @@ impl UntaggedValue { UntaggedValue::Primitive(Primitive::Binary(binary)) } + pub fn range( + left: (Spanned, RangeInclusion), + right: (Spanned, RangeInclusion), + ) -> UntaggedValue { + UntaggedValue::Primitive(Primitive::Range(Box::new(Range::new(left, right)))) + } + pub fn boolean(s: impl Into) -> UntaggedValue { UntaggedValue::Primitive(Primitive::Boolean(s.into())) } @@ -224,6 +233,23 @@ impl Value { _ => Err(ShellError::type_error("Path", self.spanned_type_name())), } } + + pub fn as_primitive(&self) -> Result { + match &self.value { + UntaggedValue::Primitive(primitive) => Ok(primitive.clone()), + _ => Err(ShellError::type_error( + "Primitive", + self.spanned_type_name(), + )), + } + } + + pub fn as_u64(&self) -> Result { + match &self.value { + UntaggedValue::Primitive(primitive) => primitive.as_u64(self.tag.span), + _ => Err(ShellError::type_error("integer", self.spanned_type_name())), + } + } } impl Into for &str { diff --git a/crates/nu-protocol/src/value/debug.rs b/crates/nu-protocol/src/value/debug.rs index f71be77124..f089b82b2c 100644 --- a/crates/nu-protocol/src/value/debug.rs +++ b/crates/nu-protocol/src/value/debug.rs @@ -28,6 +28,7 @@ impl PrettyType for Primitive { match self { Primitive::Nothing => ty("nothing"), Primitive::Int(_) => ty("integer"), + Primitive::Range(_) => ty("range"), Primitive::Decimal(_) => ty("decimal"), Primitive::Bytes(_) => ty("bytesize"), Primitive::String(_) => ty("string"), @@ -51,6 +52,21 @@ impl PrettyDebug for Primitive { Primitive::Nothing => b::primitive("nothing"), Primitive::Int(int) => prim(format_args!("{}", int)), Primitive::Decimal(decimal) => prim(format_args!("{}", decimal)), + Primitive::Range(range) => { + let (left, left_inclusion) = &range.from; + let (right, right_inclusion) = &range.to; + + b::typed( + "range", + (left_inclusion.debug_left_bracket() + + left.pretty() + + b::operator(",") + + b::space() + + right.pretty() + + right_inclusion.debug_right_bracket()) + .group(), + ) + } Primitive::Bytes(bytes) => primitive_doc(bytes, "bytesize"), Primitive::String(string) => prim(string), Primitive::Line(string) => prim(string), diff --git a/crates/nu-protocol/src/value/primitive.rs b/crates/nu-protocol/src/value/primitive.rs index 92bb3ee05d..30bf7c5222 100644 --- a/crates/nu-protocol/src/value/primitive.rs +++ b/crates/nu-protocol/src/value/primitive.rs @@ -1,12 +1,14 @@ use crate::type_name::ShellTypeName; use crate::value::column_path::ColumnPath; +use crate::value::range::Range; use crate::value::{serde_bigdecimal, serde_bigint}; use bigdecimal::BigDecimal; use chrono::{DateTime, Utc}; use chrono_humanize::Humanize; -use nu_source::PrettyDebug; +use nu_errors::{ExpectedRange, ShellError}; +use nu_source::{PrettyDebug, Span, SpannedItem}; use num_bigint::BigInt; -use num_traits::cast::FromPrimitive; +use num_traits::cast::{FromPrimitive, ToPrimitive}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -25,6 +27,7 @@ pub enum Primitive { Boolean(bool), Date(DateTime), Duration(u64), // Duration in seconds + Range(Box), Path(PathBuf), #[serde(with = "serde_bytes")] Binary(Vec), @@ -34,6 +37,25 @@ pub enum Primitive { EndOfStream, } +impl Primitive { + pub fn as_u64(&self, span: Span) -> Result { + match self { + Primitive::Int(int) => match int.to_u64() { + None => Err(ShellError::range_error( + ExpectedRange::U64, + &format!("{}", int).spanned(span), + "converting an integer into a 64-bit integer", + )), + Some(num) => Ok(num), + }, + other => Err(ShellError::type_error( + "integer", + other.type_name().spanned(span), + )), + } + } +} + impl From for Primitive { fn from(decimal: BigDecimal) -> Primitive { Primitive::Decimal(decimal) @@ -51,6 +73,7 @@ impl ShellTypeName for Primitive { match self { Primitive::Nothing => "nothing", Primitive::Int(_) => "integer", + Primitive::Range(_) => "range", Primitive::Decimal(_) => "decimal", Primitive::Bytes(_) => "bytes", Primitive::String(_) => "string", @@ -91,6 +114,11 @@ pub fn format_primitive(primitive: &Primitive, field_name: Option<&String>) -> S Primitive::Duration(sec) => format_duration(*sec), Primitive::Int(i) => i.to_string(), Primitive::Decimal(decimal) => decimal.to_string(), + Primitive::Range(range) => format!( + "{}..{}", + format_primitive(&range.from.0.item, None), + format_primitive(&range.to.0.item, None) + ), Primitive::Pattern(s) => s.to_string(), Primitive::String(s) => s.to_owned(), Primitive::Line(s) => s.to_owned(), @@ -125,7 +153,8 @@ pub fn format_primitive(primitive: &Primitive, field_name: Option<&String>) -> S Primitive::Date(d) => d.humanize().to_string(), } } -fn format_duration(sec: u64) -> String { + +pub fn format_duration(sec: u64) -> String { let (minutes, seconds) = (sec / 60, sec % 60); let (hours, minutes) = (minutes / 60, minutes % 60); let (days, hours) = (hours / 24, hours % 24); diff --git a/crates/nu-protocol/src/value/range.rs b/crates/nu-protocol/src/value/range.rs new file mode 100644 index 0000000000..f1f9a90fa8 --- /dev/null +++ b/crates/nu-protocol/src/value/range.rs @@ -0,0 +1,32 @@ +use crate::value::Primitive; +use derive_new::new; +use nu_source::{b, DebugDocBuilder, Spanned}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize, Hash)] +pub enum RangeInclusion { + Inclusive, + Exclusive, +} + +impl RangeInclusion { + pub fn debug_left_bracket(&self) -> DebugDocBuilder { + b::delimiter(match self { + RangeInclusion::Exclusive => "(", + RangeInclusion::Inclusive => "[", + }) + } + + pub fn debug_right_bracket(&self) -> DebugDocBuilder { + b::delimiter(match self { + RangeInclusion::Exclusive => ")", + RangeInclusion::Inclusive => "]", + }) + } +} + +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize, new)] +pub struct Range { + pub from: (Spanned, RangeInclusion), + pub to: (Spanned, RangeInclusion), +} diff --git a/crates/nu-source/src/meta.rs b/crates/nu-source/src/meta.rs index 70012cc635..bca7135da9 100644 --- a/crates/nu-source/src/meta.rs +++ b/crates/nu-source/src/meta.rs @@ -220,10 +220,7 @@ impl nom_locate::LocatedSpanEx, ), ) -> Span { - Span { - start: input.0.offset, - end: input.1.offset, - } + Span::new(input.0.offset, input.1.offset) } } @@ -235,10 +232,7 @@ impl From<(usize, usize)> for Span { impl From<&std::ops::Range> for Span { fn from(input: &std::ops::Range) -> Span { - Span { - start: input.start, - end: input.end, - } + Span::new(input.start, input.end) } } @@ -321,10 +315,7 @@ impl Tag { pub fn for_char(pos: usize, anchor: AnchorLocation) -> Tag { Tag { anchor: Some(anchor), - span: Span { - start: pos, - end: pos + 1, - }, + span: Span::new(pos, pos + 1), } } @@ -528,11 +519,19 @@ impl Span { impl language_reporting::ReportingSpan for Span { fn with_start(&self, start: usize) -> Self { - Span::new(start, self.end) + if self.end < start { + Span::new(start, start) + } else { + Span::new(start, self.end) + } } fn with_end(&self, end: usize) -> Self { - Span::new(self.start, end) + if end < self.start { + Span::new(end, end) + } else { + Span::new(self.start, end) + } } fn start(&self) -> usize { diff --git a/crates/nu_plugin_post/src/main.rs b/crates/nu_plugin_post/src/main.rs index 8e802a2995..8a5bd606ad 100644 --- a/crates/nu_plugin_post/src/main.rs +++ b/crates/nu_plugin_post/src/main.rs @@ -447,7 +447,9 @@ pub fn value_to_json_value(v: &Value) -> Result { UntaggedValue::Table(l) => serde_json::Value::Array(json_list(l)?), UntaggedValue::Error(e) => return Err(e.clone()), - UntaggedValue::Block(_) => serde_json::Value::Null, + UntaggedValue::Block(_) | UntaggedValue::Primitive(Primitive::Range(_)) => { + serde_json::Value::Null + } UntaggedValue::Primitive(Primitive::Binary(b)) => serde_json::Value::Array( b.iter() .map(|x| { diff --git a/src/commands/cd.rs b/src/commands/cd.rs index b96619caed..a57356d7ce 100644 --- a/src/commands/cd.rs +++ b/src/commands/cd.rs @@ -1,6 +1,7 @@ use crate::commands::WholeStreamCommand; use crate::prelude::*; use nu_errors::ShellError; +use nu_macros::signature; use nu_protocol::{Signature, SyntaxShape}; pub struct CD; @@ -11,11 +12,17 @@ impl WholeStreamCommand for CD { } fn signature(&self) -> Signature { - Signature::build("cd").optional( - "directory", - SyntaxShape::Path, - "the directory to change to", - ) + signature! { + def cd { + "the directory to change to" + directory(optional Path) - "the directory to change to" + } + } + // Signature::build("cd").optional( + // "directory", + // SyntaxShape::Path, + // "the directory to change to", + // ) } fn usage(&self) -> &str { diff --git a/src/commands/command.rs b/src/commands/command.rs index ef5443a3ed..5873d5ea20 100644 --- a/src/commands/command.rs +++ b/src/commands/command.rs @@ -378,14 +378,7 @@ pub trait WholeStreamCommand: Send + Sync { fn name(&self) -> &str; fn signature(&self) -> Signature { - Signature { - name: self.name().to_string(), - usage: self.usage().to_string(), - positional: vec![], - rest_positional: None, - named: indexmap::IndexMap::new(), - is_filter: true, - } + Signature::new(self.name()).desc(self.usage()).filter() } fn usage(&self) -> &str; @@ -405,14 +398,7 @@ pub trait PerItemCommand: Send + Sync { fn name(&self) -> &str; fn signature(&self) -> Signature { - Signature { - name: self.name().to_string(), - usage: self.usage().to_string(), - positional: vec![], - rest_positional: None, - named: indexmap::IndexMap::new(), - is_filter: true, - } + Signature::new(self.name()).desc(self.usage()).filter() } fn usage(&self) -> &str; diff --git a/src/commands/range.rs b/src/commands/range.rs index edb355451e..c2e9a14d83 100644 --- a/src/commands/range.rs +++ b/src/commands/range.rs @@ -1,5 +1,6 @@ use crate::commands::WholeStreamCommand; use crate::context::CommandRegistry; +use crate::deserializer::NumericRange; use crate::prelude::*; use nu_errors::ShellError; use nu_protocol::{Signature, SyntaxShape}; @@ -7,7 +8,7 @@ use nu_source::Tagged; #[derive(Deserialize)] struct RangeArgs { - area: Tagged, + area: Tagged, } pub struct Range; @@ -20,7 +21,7 @@ impl WholeStreamCommand for Range { fn signature(&self) -> Signature { Signature::build("range").required( "rows ", - SyntaxShape::Any, + SyntaxShape::Range, "range of rows to return: Eg) 4..7 (=> from 4 to 7)", ) } @@ -39,48 +40,14 @@ impl WholeStreamCommand for Range { } fn range( - RangeArgs { area: rows }: RangeArgs, - RunnableContext { input, name, .. }: RunnableContext, + RangeArgs { area }: RangeArgs, + RunnableContext { input, name: _, .. }: RunnableContext, ) -> Result { - match rows.item.find('.') { - Some(value) => { - let (first, last) = rows.item.split_at(value); - let first = match first.parse::() { - Ok(postion) => postion, - Err(_) => { - if first == "" { - 0 - } else { - return Err(ShellError::labeled_error( - "no correct start of range", - "'from' needs to be an Integer or empty", - name, - )); - } - } - }; - let last = match last.trim_start_matches('.').parse::() { - Ok(postion) => postion, - Err(_) => { - if last == ".." { - std::u64::MAX - 1 - } else { - return Err(ShellError::labeled_error( - "no correct end of range", - "'to' needs to be an Integer or empty", - name, - )); - } - } - }; - Ok(OutputStream::from_input( - input.values.skip(first).take(last - first + 1), - )) - } - None => Err(ShellError::labeled_error( - "No correct formatted range found", - "format: ..", - name, - )), - } + let range = area.item; + let (from, _) = range.from; + let (to, _) = range.to; + + return Ok(OutputStream::from_input( + input.values.skip(*from).take(*to - *from + 1), + )); } diff --git a/src/commands/to_bson.rs b/src/commands/to_bson.rs index 3fdc7e89b9..3b783b7af4 100644 --- a/src/commands/to_bson.rs +++ b/src/commands/to_bson.rs @@ -73,7 +73,7 @@ pub fn value_to_bson_value(v: &Value) -> Result { .map(|x| value_to_bson_value(x)) .collect::>()?, ), - UntaggedValue::Block(_) => Bson::Null, + UntaggedValue::Block(_) | UntaggedValue::Primitive(Primitive::Range(_)) => Bson::Null, UntaggedValue::Error(e) => return Err(e.clone()), UntaggedValue::Primitive(Primitive::Binary(b)) => { Bson::Binary(BinarySubtype::Generic, b.clone()) diff --git a/src/commands/to_json.rs b/src/commands/to_json.rs index dfa4aef5c1..929fa8f195 100644 --- a/src/commands/to_json.rs +++ b/src/commands/to_json.rs @@ -76,7 +76,9 @@ pub fn value_to_json_value(v: &Value) -> Result { UntaggedValue::Table(l) => serde_json::Value::Array(json_list(l)?), UntaggedValue::Error(e) => return Err(e.clone()), - UntaggedValue::Block(_) => serde_json::Value::Null, + UntaggedValue::Block(_) | UntaggedValue::Primitive(Primitive::Range(_)) => { + serde_json::Value::Null + } UntaggedValue::Primitive(Primitive::Binary(b)) => serde_json::Value::Array( b.iter() .map(|x| { diff --git a/src/commands/to_sqlite.rs b/src/commands/to_sqlite.rs index 6b76a7cb0f..9f6ef5546e 100644 --- a/src/commands/to_sqlite.rs +++ b/src/commands/to_sqlite.rs @@ -100,9 +100,10 @@ fn nu_value_to_sqlite_string(v: Value) -> String { Primitive::Date(d) => format!("'{}'", d), Primitive::Path(p) => format!("'{}'", p.display().to_string().replace("'", "''")), Primitive::Binary(u) => format!("x'{}'", encode(u)), - Primitive::BeginningOfStream | Primitive::EndOfStream | Primitive::ColumnPath(_) => { - "NULL".into() - } + Primitive::BeginningOfStream + | Primitive::EndOfStream + | Primitive::ColumnPath(_) + | Primitive::Range(_) => "NULL".into(), }, _ => "NULL".into(), } diff --git a/src/commands/to_toml.rs b/src/commands/to_toml.rs index 351602677a..48235723b3 100644 --- a/src/commands/to_toml.rs +++ b/src/commands/to_toml.rs @@ -69,6 +69,7 @@ pub fn value_to_toml_value(v: &Value) -> Result { UntaggedValue::Table(l) => toml::Value::Array(collect_values(l)?), UntaggedValue::Error(e) => return Err(e.clone()), UntaggedValue::Block(_) => toml::Value::String("".to_string()), + UntaggedValue::Primitive(Primitive::Range(_)) => toml::Value::String("".to_string()), UntaggedValue::Primitive(Primitive::Binary(b)) => { toml::Value::Array(b.iter().map(|x| toml::Value::Integer(*x as i64)).collect()) } diff --git a/src/commands/to_yaml.rs b/src/commands/to_yaml.rs index aa21108d99..f0c1c0da10 100644 --- a/src/commands/to_yaml.rs +++ b/src/commands/to_yaml.rs @@ -85,7 +85,9 @@ pub fn value_to_yaml_value(v: &Value) -> Result { serde_yaml::Value::Sequence(out) } UntaggedValue::Error(e) => return Err(e.clone()), - UntaggedValue::Block(_) => serde_yaml::Value::Null, + UntaggedValue::Block(_) | UntaggedValue::Primitive(Primitive::Range(_)) => { + serde_yaml::Value::Null + } UntaggedValue::Primitive(Primitive::Binary(b)) => serde_yaml::Value::Sequence( b.iter() .map(|x| serde_yaml::Value::Number(serde_yaml::Number::from(*x))) diff --git a/src/data/base.rs b/src/data/base.rs index f30397a956..4fdee475dc 100644 --- a/src/data/base.rs +++ b/src/data/base.rs @@ -7,7 +7,7 @@ use chrono::{DateTime, Utc}; use derive_new::new; use log::trace; use nu_errors::ShellError; -use nu_parser::{hir, Operator}; +use nu_parser::{hir, CompareOperator}; use nu_protocol::{ Evaluate, EvaluateTrait, Primitive, Scope, ShellTypeName, SpannedTypeName, TaggedDictBuilder, UntaggedValue, Value, @@ -23,7 +23,7 @@ use std::time::SystemTime; #[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, new, Serialize)] pub struct Operation { pub(crate) left: Value, - pub(crate) operator: Operator, + pub(crate) operator: CompareOperator, pub(crate) right: Value, } diff --git a/src/data/base/shape.rs b/src/data/base/shape.rs index 44521c772a..f6ffac0e7d 100644 --- a/src/data/base/shape.rs +++ b/src/data/base/shape.rs @@ -1,207 +1,24 @@ use crate::prelude::*; use chrono::{DateTime, Utc}; use chrono_humanize::Humanize; -use derive_new::new; use indexmap::IndexMap; use nu_errors::ShellError; +use nu_protocol::RangeInclusion; use nu_protocol::{ format_primitive, ColumnPath, Dictionary, Evaluate, Primitive, ShellTypeName, TaggedDictBuilder, UntaggedValue, Value, }; -use nu_source::{b, DebugDoc, PrettyDebug}; +use nu_source::{b, PrettyDebug}; use std::collections::BTreeMap; use std::fmt::Debug; use std::hash::Hash; use std::io::Write; use std::path::PathBuf; -/** - This file describes the structural types of the nushell system. - - Its primary purpose today is to identify "equivalent" values for the purpose - of merging rows into a single table or identify rows in a table that have the - same shape for reflection. - - It also serves as the primary vehicle for pretty-printing. -*/ - -#[allow(unused)] #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum TypeShape { - Nothing, - Int, - Decimal, - Bytesize, - String, - Line, - ColumnPath, - Pattern, - Boolean, - Date, - Duration, - Path, - Binary, - - Row(BTreeMap), - Table(Vec), - - // TODO: Block arguments - Block, - // TODO: Error type - Error, - - // Stream markers (used as bookend markers rather than actual values) - BeginningOfStream, - EndOfStream, -} - -impl TypeShape { - pub fn from_primitive(primitive: &Primitive) -> TypeShape { - match primitive { - Primitive::Nothing => TypeShape::Nothing, - Primitive::Int(_) => TypeShape::Int, - Primitive::Decimal(_) => TypeShape::Decimal, - Primitive::Bytes(_) => TypeShape::Bytesize, - Primitive::String(_) => TypeShape::String, - Primitive::Line(_) => TypeShape::Line, - Primitive::ColumnPath(_) => TypeShape::ColumnPath, - Primitive::Pattern(_) => TypeShape::Pattern, - Primitive::Boolean(_) => TypeShape::Boolean, - Primitive::Date(_) => TypeShape::Date, - Primitive::Duration(_) => TypeShape::Duration, - Primitive::Path(_) => TypeShape::Path, - Primitive::Binary(_) => TypeShape::Binary, - Primitive::BeginningOfStream => TypeShape::BeginningOfStream, - Primitive::EndOfStream => TypeShape::EndOfStream, - } - } - - pub fn from_dictionary(dictionary: &Dictionary) -> TypeShape { - let mut map = BTreeMap::new(); - - for (key, value) in dictionary.entries.iter() { - let column = Column::String(key.clone()); - map.insert(column, TypeShape::from_value(value)); - } - - TypeShape::Row(map) - } - - pub fn from_table<'a>(table: impl IntoIterator) -> TypeShape { - let mut vec = vec![]; - - for item in table.into_iter() { - vec.push(TypeShape::from_value(item)) - } - - TypeShape::Table(vec) - } - - pub fn from_value<'a>(value: impl Into<&'a UntaggedValue>) -> TypeShape { - match value.into() { - UntaggedValue::Primitive(p) => TypeShape::from_primitive(p), - UntaggedValue::Row(row) => TypeShape::from_dictionary(row), - UntaggedValue::Table(table) => TypeShape::from_table(table.iter()), - UntaggedValue::Error(_) => TypeShape::Error, - UntaggedValue::Block(_) => TypeShape::Block, - } - } -} - -impl PrettyDebug for TypeShape { - fn pretty(&self) -> DebugDocBuilder { - match self { - TypeShape::Nothing => ty("nothing"), - TypeShape::Int => ty("integer"), - TypeShape::Decimal => ty("decimal"), - TypeShape::Bytesize => ty("bytesize"), - TypeShape::String => ty("string"), - TypeShape::Line => ty("line"), - TypeShape::ColumnPath => ty("column-path"), - TypeShape::Pattern => ty("pattern"), - TypeShape::Boolean => ty("boolean"), - TypeShape::Date => ty("date"), - TypeShape::Duration => ty("duration"), - TypeShape::Path => ty("path"), - TypeShape::Binary => ty("binary"), - TypeShape::Error => b::error("error"), - TypeShape::BeginningOfStream => b::keyword("beginning-of-stream"), - TypeShape::EndOfStream => b::keyword("end-of-stream"), - TypeShape::Row(row) => (b::kind("row") - + b::space() - + b::intersperse( - row.iter().map(|(key, ty)| { - (b::key(match key { - Column::String(string) => string.clone(), - Column::Value => "".to_string(), - }) + b::delimit("(", ty.pretty(), ")").into_kind()) - .nest() - }), - b::space(), - ) - .nest()) - .nest(), - - TypeShape::Table(table) => { - let mut group: Group> = Group::new(); - - for (i, item) in table.iter().enumerate() { - group.add(item.to_doc(), i); - } - - (b::kind("table") + b::space() + b::keyword("of")).group() - + b::space() - + (if group.len() == 1 { - let (doc, _) = group.into_iter().nth(0).unwrap(); - DebugDocBuilder::from_doc(doc) - } else { - b::intersperse( - group.into_iter().map(|(doc, rows)| { - (b::intersperse( - rows.iter().map(|(from, to)| { - if from == to { - b::description(from) - } else { - (b::description(from) - + b::space() - + b::keyword("to") - + b::space() - + b::description(to)) - .group() - } - }), - b::description(", "), - ) + b::description(":") - + b::space() - + DebugDocBuilder::from_doc(doc)) - .nest() - }), - b::space(), - ) - }) - } - TypeShape::Block => ty("block"), - } - } -} - -#[derive(Debug, new)] -struct DebugEntry<'a> { - key: &'a Column, - value: &'a TypeShape, -} - -impl<'a> PrettyDebug for DebugEntry<'a> { - fn pretty(&self) -> DebugDocBuilder { - (b::key(match self.key { - Column::String(string) => string.clone(), - Column::Value => "".to_owned(), - }) + b::delimit("(", self.value.pretty(), ")").into_kind()) - } -} - -fn ty(name: impl std::fmt::Display) -> DebugDocBuilder { - b::kind(format!("{}", name)) +pub struct InlineRange { + from: (InlineShape, RangeInclusion), + to: (InlineShape, RangeInclusion), } #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] @@ -209,6 +26,7 @@ pub enum InlineShape { Nothing, Int(BigInt), Decimal(BigDecimal), + Range(Box), Bytesize(u64), String(String), Line(String), @@ -243,6 +61,15 @@ impl InlineShape { match primitive { Primitive::Nothing => InlineShape::Nothing, Primitive::Int(int) => InlineShape::Int(int.clone()), + Primitive::Range(range) => { + let (left, left_inclusion) = &range.from; + let (right, right_inclusion) = &range.to; + + InlineShape::Range(Box::new(InlineRange { + from: (InlineShape::from_primitive(left), *left_inclusion), + to: (InlineShape::from_primitive(right), *right_inclusion), + })) + } Primitive::Decimal(decimal) => InlineShape::Decimal(decimal.clone()), Primitive::Bytes(bytesize) => InlineShape::Bytesize(*bytesize), Primitive::String(string) => InlineShape::String(string.clone()), @@ -314,6 +141,17 @@ impl PrettyDebug for FormatInlineShape { InlineShape::Nothing => b::blank(), InlineShape::Int(int) => b::primitive(format!("{}", int)), InlineShape::Decimal(decimal) => b::primitive(format!("{}", decimal)), + InlineShape::Range(range) => { + let (left, left_inclusion) = &range.from; + let (right, right_inclusion) = &range.to; + + let op = match (left_inclusion, right_inclusion) { + (RangeInclusion::Inclusive, RangeInclusion::Exclusive) => "..", + _ => unimplemented!("No syntax for ranges that aren't inclusive on the left and exclusive on the right") + }; + + left.clone().format().pretty() + b::operator(op) + right.clone().format().pretty() + } InlineShape::Bytesize(bytesize) => { let byte = byte_unit::Byte::from_bytes(*bytesize as u128); @@ -411,51 +249,6 @@ impl GroupedValue for Vec<(usize, usize)> { } } -#[derive(Debug)] -pub struct Group { - values: indexmap::IndexMap, -} - -impl Group -where - K: Debug + Eq + Hash, - G: GroupedValue, -{ - pub fn new() -> Group { - Group { - values: indexmap::IndexMap::default(), - } - } - - pub fn len(&self) -> usize { - self.values.len() - } - - pub fn into_iter(self) -> impl Iterator { - self.values.into_iter() - } - - pub fn add(&mut self, key: impl Into, value: impl Into) { - let key = key.into(); - let value = value.into(); - - let group = self.values.get_mut(&key); - - match group { - None => { - self.values.insert(key, { - let mut group = G::new(); - group.merge(value); - group - }); - } - Some(group) => { - group.merge(value); - } - } - } -} - #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum Column { String(String), diff --git a/src/data/value.rs b/src/data/value.rs index 8c8da375b8..aa1eb3d521 100644 --- a/src/data/value.rs +++ b/src/data/value.rs @@ -1,10 +1,10 @@ use crate::data::base::coerce_compare; -use crate::data::base::shape::{Column, InlineShape, TypeShape}; +use crate::data::base::shape::{Column, InlineShape}; use crate::data::primitive::style_primitive; use chrono::DateTime; use nu_errors::ShellError; -use nu_parser::Operator; -use nu_protocol::{Primitive, UntaggedValue}; +use nu_parser::CompareOperator; +use nu_protocol::{Primitive, Type, UntaggedValue}; use nu_source::{DebugDocBuilder, PrettyDebug, Tagged}; pub fn date_from_str(s: Tagged<&str>) -> Result { @@ -22,7 +22,7 @@ pub fn date_from_str(s: Tagged<&str>) -> Result { } pub fn compare_values( - operator: Operator, + operator: &CompareOperator, left: &UntaggedValue, right: &UntaggedValue, ) -> Result { @@ -34,16 +34,15 @@ pub fn compare_values( use std::cmp::Ordering; let result = match (operator, ordering) { - (Operator::Equal, Ordering::Equal) => true, - (Operator::NotEqual, Ordering::Less) | (Operator::NotEqual, Ordering::Greater) => { - true - } - (Operator::LessThan, Ordering::Less) => true, - (Operator::GreaterThan, Ordering::Greater) => true, - (Operator::GreaterThanOrEqual, Ordering::Greater) - | (Operator::GreaterThanOrEqual, Ordering::Equal) => true, - (Operator::LessThanOrEqual, Ordering::Less) - | (Operator::LessThanOrEqual, Ordering::Equal) => true, + (CompareOperator::Equal, Ordering::Equal) => true, + (CompareOperator::NotEqual, Ordering::Less) + | (CompareOperator::NotEqual, Ordering::Greater) => true, + (CompareOperator::LessThan, Ordering::Less) => true, + (CompareOperator::GreaterThan, Ordering::Greater) => true, + (CompareOperator::GreaterThanOrEqual, Ordering::Greater) + | (CompareOperator::GreaterThanOrEqual, Ordering::Equal) => true, + (CompareOperator::LessThanOrEqual, Ordering::Less) + | (CompareOperator::LessThanOrEqual, Ordering::Equal) => true, _ => false, }; @@ -53,7 +52,7 @@ pub fn compare_values( } pub fn format_type<'a>(value: impl Into<&'a UntaggedValue>, width: usize) -> String { - TypeShape::from_value(value.into()).colored_string(width) + Type::from_value(value.into()).colored_string(width) } pub fn format_leaf<'a>(value: impl Into<&'a UntaggedValue>) -> DebugDocBuilder { diff --git a/src/deserializer.rs b/src/deserializer.rs index 0b23b81657..b82ad64a1f 100644 --- a/src/deserializer.rs +++ b/src/deserializer.rs @@ -1,11 +1,20 @@ use log::trace; use nu_errors::{CoerceInto, ShellError}; -use nu_protocol::{CallInfo, ColumnPath, Evaluate, Primitive, ShellTypeName, UntaggedValue, Value}; -use nu_source::{HasSpan, SpannedItem, Tagged, TaggedItem}; +use nu_protocol::{ + CallInfo, ColumnPath, Evaluate, Primitive, RangeInclusion, ShellTypeName, UntaggedValue, Value, +}; +use nu_source::{HasSpan, Spanned, SpannedItem, Tagged, TaggedItem}; use nu_value_ext::ValueExt; use serde::de; +use serde::{Deserialize, Serialize}; use std::path::PathBuf; +#[derive(Copy, Clone, Deserialize, Serialize)] +pub struct NumericRange { + pub from: (Spanned, RangeInclusion), + pub to: (Spanned, RangeInclusion), +} + #[derive(Debug)] pub struct DeserializerItem<'de> { key_struct_field: Option<(String, &'de str)>, @@ -406,6 +415,25 @@ impl<'de, 'a> de::Deserializer<'de> for &'a mut ConfigDeserializer<'de> { value: UntaggedValue::Primitive(Primitive::String(string)), .. } => visit::, _>(string.tagged(tag), name, fields, visitor), + Value { + value: UntaggedValue::Primitive(Primitive::Range(range)), + .. + } => { + let (left, left_inclusion) = range.from; + let (right, right_inclusion) = range.to; + let left_span = left.span; + let right_span = right.span; + + let left = left.as_u64(left_span)?; + let right = right.as_u64(right_span)?; + + let numeric_range = NumericRange { + from: (left.spanned(left_span), left_inclusion), + to: (right.spanned(right_span), right_inclusion), + }; + + visit::, _>(numeric_range.tagged(tag), name, fields, visitor) + } other => Err(ShellError::type_error( name, diff --git a/src/evaluate/evaluator.rs b/src/evaluate/evaluator.rs index c3f89dfd38..03808f2a84 100644 --- a/src/evaluate/evaluator.rs +++ b/src/evaluate/evaluator.rs @@ -6,8 +6,8 @@ use log::trace; use nu_errors::{ArgumentError, ShellError}; use nu_parser::hir::{self, Expression, RawExpression}; use nu_protocol::{ - ColumnPath, Evaluate, Primitive, Scope, TaggedDictBuilder, UnspannedPathMember, UntaggedValue, - Value, + ColumnPath, Evaluate, Primitive, RangeInclusion, Scope, TaggedDictBuilder, UnspannedPathMember, + UntaggedValue, Value, }; use nu_source::Text; @@ -40,7 +40,7 @@ pub(crate) fn evaluate_baseline_expr( trace!("left={:?} right={:?}", left.value, right.value); - match apply_operator(**binary.op(), &left, &right) { + match apply_operator(&**binary.op(), &left, &right) { Ok(result) => Ok(result.into_value(tag)), Err((left_type, right_type)) => Err(ShellError::coerce_error( left_type.spanned(binary.left().span), @@ -48,6 +48,26 @@ pub(crate) fn evaluate_baseline_expr( )), } } + RawExpression::Range(range) => { + let left = range.left(); + let right = range.right(); + + let left = evaluate_baseline_expr(left, registry, scope, source)?; + let right = evaluate_baseline_expr(right, registry, scope, source)?; + let left_span = left.tag.span; + let right_span = right.tag.span; + + let left = ( + left.as_primitive()?.spanned(left_span), + RangeInclusion::Inclusive, + ); + let right = ( + right.as_primitive()?.spanned(right_span), + RangeInclusion::Exclusive, + ); + + Ok(UntaggedValue::range(left, right).into_value(tag)) + } RawExpression::List(list) => { let mut exprs = vec![]; diff --git a/src/evaluate/operator.rs b/src/evaluate/operator.rs index e96e6fd0fc..7eee06051b 100644 --- a/src/evaluate/operator.rs +++ b/src/evaluate/operator.rs @@ -1,25 +1,24 @@ use crate::data::value; -use nu_parser::Operator; +use nu_parser::CompareOperator; use nu_protocol::{Primitive, ShellTypeName, UntaggedValue, Value}; use std::ops::Not; pub fn apply_operator( - op: Operator, + op: &CompareOperator, left: &Value, right: &Value, ) -> Result { - match op { - Operator::Equal - | Operator::NotEqual - | Operator::LessThan - | Operator::GreaterThan - | Operator::LessThanOrEqual - | Operator::GreaterThanOrEqual => { + match *op { + CompareOperator::Equal + | CompareOperator::NotEqual + | CompareOperator::LessThan + | CompareOperator::GreaterThan + | CompareOperator::LessThanOrEqual + | CompareOperator::GreaterThanOrEqual => { value::compare_values(op, left, right).map(UntaggedValue::boolean) } - Operator::Dot => Ok(UntaggedValue::boolean(false)), - Operator::Contains => contains(left, right).map(UntaggedValue::boolean), - Operator::NotContains => contains(left, right) + CompareOperator::Contains => contains(left, right).map(UntaggedValue::boolean), + CompareOperator::NotContains => contains(left, right) .map(Not::not) .map(UntaggedValue::boolean), } diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 3008d5514d..8ad1b29771 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -144,7 +144,8 @@ fn paint_flat_shape(flat_shape: &Spanned, line: &str) -> String { FlatShape::CloseDelimiter(_) => Color::White.normal(), FlatShape::ItVariable => Color::Purple.bold(), FlatShape::Variable => Color::Purple.normal(), - FlatShape::Operator => Color::Yellow.normal(), + FlatShape::CompareOperator => Color::Yellow.normal(), + FlatShape::DotDot => Color::Yellow.bold(), FlatShape::Dot => Color::White.normal(), FlatShape::InternalCommand => Color::Cyan.bold(), FlatShape::ExternalCommand => Color::Cyan.normal(), @@ -160,7 +161,8 @@ fn paint_flat_shape(flat_shape: &Spanned, line: &str) -> String { FlatShape::ShorthandFlag => Color::Black.bold(), FlatShape::Int => Color::Purple.bold(), FlatShape::Decimal => Color::Purple.bold(), - FlatShape::Whitespace => Color::White.normal(), + FlatShape::Whitespace | FlatShape::Separator => Color::White.normal(), + FlatShape::Comment => Color::Black.bold(), FlatShape::Error => Color::Red.bold(), FlatShape::Size { number, unit } => { let number = number.slice(line); diff --git a/tests/command_cd_tests.rs b/tests/command_cd_tests.rs index 8b6592c940..1f67bf7fa9 100644 --- a/tests/command_cd_tests.rs +++ b/tests/command_cd_tests.rs @@ -130,8 +130,8 @@ fn filesystem_not_a_directory() { "cd ferris_did_it.txt" ); - assert!(actual.contains("ferris_did_it.txt")); - assert!(actual.contains("is not a directory")); + assert!(actual.contains("ferris_did_it.txt"), "actual={:?}", actual); + assert!(actual.contains("is not a directory"), "actual={:?}", actual); }) } @@ -142,8 +142,16 @@ fn filesystem_directory_not_found() { "cd dir_that_does_not_exist" ); - assert!(actual.contains("dir_that_does_not_exist")); - assert!(actual.contains("directory not found")); + assert!( + actual.contains("dir_that_does_not_exist"), + "actual={:?}", + actual + ); + assert!( + actual.contains("directory not found"), + "actual={:?}", + actual + ); } #[test] diff --git a/tests/commands_test.rs b/tests/commands_test.rs index 2d40cd7313..7b607830a1 100644 --- a/tests/commands_test.rs +++ b/tests/commands_test.rs @@ -248,30 +248,6 @@ fn range_selects_some_rows() { }); } -#[test] -fn range_selects_all_rows() { - Playground::setup("range_test_3", |dirs, sandbox| { - sandbox.with_files(vec![ - EmptyFile("notes.txt"), - EmptyFile("tests.txt"), - EmptyFile("persons.txt"), - ]); - - let actual = nu!( - cwd: dirs.test(), h::pipeline( - r#" - ls - | get name - | range .. - | count - | echo $it - "# - )); - - assert_eq!(actual, "3"); - }); -} - #[test] fn split_by() { Playground::setup("split_by_test_1", |dirs, sandbox| {