b150f9f5d8
# Description Fixes: #8565 Here is another pr #7240 tried to address the issue, but it works in a wrong way. After this change `o+e>` won't redirect all stdout message then stderr message and it works more like how bash does. # User-Facing Changes For the given python code: ```python # test.py import sys print('aa'*300, flush=True) print('bb'*999999, file=sys.stderr, flush=True) print('cc'*300, flush=True) ``` Running `python test.py out+err> a.txt` shoudn't hang nushell, and `a.txt` keeps output in the same order ## About the change The core idea is that when doing lite-parsing, introduce a new variant `LiteElement::SameTargetRedirection` if we meet `out+err>` redirection token(which is generated by lex function), During converting from lite block to block, LiteElement::SameTargetRedirection will be converted to PipelineElement::SameTargetRedirection. Then in the block eval process, if we get PipelineElement::SameTargetRedirection, we'll invoke `run-external` with `--redirect-combine` flag, then pipe the result into save command ## What happened internally? Take the following command as example: `^ls o+e> log.txt` lex parsing result(`Tokens`) are not changed, but `LiteBlock` and `Block` is changed after this pr. ### LiteBlock before ```rust LiteBlock { block: [ LitePipeline { commands: [ Command(None, LiteCommand { comments: [], parts: [Span { start: 39041, end: 39044 }] }), // actually the span of first Redirection is wrong too.. Redirection(Span { start: 39058, end: 39062 }, StdoutAndStderr, LiteCommand { comments: [], parts: [Span { start: 39050, end: 39057 }] }), ] }] } ``` ### LiteBlock after ```rust LiteBlock { block: [ LitePipeline { commands: [ SameTargetRedirection { cmd: (None, LiteCommand { comments: [], parts: [Span { start: 147945, end: 147948}]}), redirection: (Span { start: 147949, end: 147957 }, LiteCommand { comments: [], parts: [Span { start: 147958, end: 147965 }]}) } ] } ] } ``` ### Block before ```rust Pipeline { elements: [ Expression(None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 39042, end: 39044 }, ty: String, custom_completion: None }, [], false), span: Span { start: 39041, end: 39044 }, ty: Any, custom_completion: None }), Redirection(Span { start: 39058, end: 39062 }, StdoutAndStderr, Expression { expr: String("out.txt"), span: Span { start: 39050, end: 39057 }, ty: String, custom_completion: None })] } ``` ### Block after ```rust Pipeline { elements: [ SameTargetRedirection { cmd: (None, Expression { expr: ExternalCall(Expression { expr: String("ls"), span: Span { start: 147946, end: 147948 }, ty: String, custom_completion: None}, [], false), span: Span { start: 147945, end: 147948}, ty: Any, custom_completion: None }), redirection: (Span { start: 147949, end: 147957}, Expression {expr: String("log.txt"), span: Span { start: 147958, end: 147965 },ty: String,custom_completion: None} } ] } ``` # Tests + Formatting Don't forget to add tests that cover your changes. Make sure you've run and fixed any issues with these commands: - `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes) - `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used -A clippy::needless_collect` to check that you're using the standard code style - `cargo test --workspace` to check that all tests pass - `cargo run -- crates/nu-utils/standard_library/tests.nu` to run the tests for the standard library > **Note** > from `nushell` you can also use the `toolkit` as follows > ```bash > use toolkit.nu # or use an `env_change` hook to activate it automatically > toolkit check pr > ``` # After Submitting If your PR had any user-facing changes, update [the documentation](https://github.com/nushell/nushell.github.io) after the PR is merged, if necessary. This will help us keep the docs up to date.
562 lines
20 KiB
Rust
562 lines
20 KiB
Rust
use nu_protocol::ast::{Call, Expr, Expression, PipelineElement};
|
|
use nu_protocol::engine::{Command, EngineState, Stack, StateWorkingSet};
|
|
use nu_protocol::{
|
|
Category, Example, IntoPipelineData, PipelineData, Range, ShellError, Signature, Span, Type,
|
|
Unit, Value,
|
|
};
|
|
#[derive(Clone)]
|
|
pub struct FromNuon;
|
|
|
|
impl Command for FromNuon {
|
|
fn name(&self) -> &str {
|
|
"from nuon"
|
|
}
|
|
|
|
fn usage(&self) -> &str {
|
|
"Convert from nuon to structured data."
|
|
}
|
|
|
|
fn signature(&self) -> nu_protocol::Signature {
|
|
Signature::build("from nuon")
|
|
.input_output_types(vec![(Type::String, Type::Any)])
|
|
.category(Category::Experimental)
|
|
}
|
|
|
|
fn examples(&self) -> Vec<Example> {
|
|
vec![
|
|
Example {
|
|
example: "'{ a:1 }' | from nuon",
|
|
description: "Converts nuon formatted string to table",
|
|
result: Some(Value::Record {
|
|
cols: vec!["a".to_string()],
|
|
vals: vec![Value::test_int(1)],
|
|
span: Span::test_data(),
|
|
}),
|
|
},
|
|
Example {
|
|
example: "'{ a:1, b: [1, 2] }' | from nuon",
|
|
description: "Converts nuon formatted string to table",
|
|
result: Some(Value::Record {
|
|
cols: vec!["a".to_string(), "b".to_string()],
|
|
vals: vec![
|
|
Value::test_int(1),
|
|
Value::List {
|
|
vals: vec![Value::test_int(1), Value::test_int(2)],
|
|
span: Span::test_data(),
|
|
},
|
|
],
|
|
span: Span::test_data(),
|
|
}),
|
|
},
|
|
]
|
|
}
|
|
|
|
fn run(
|
|
&self,
|
|
engine_state: &EngineState,
|
|
_stack: &mut Stack,
|
|
call: &Call,
|
|
input: PipelineData,
|
|
) -> Result<PipelineData, ShellError> {
|
|
let head = call.head;
|
|
let (string_input, _span, metadata) = input.collect_string_strict(head)?;
|
|
|
|
let engine_state = engine_state.clone();
|
|
|
|
let mut working_set = StateWorkingSet::new(&engine_state);
|
|
|
|
let mut block = nu_parser::parse(&mut working_set, None, string_input.as_bytes(), false);
|
|
|
|
if let Some(pipeline) = block.pipelines.get(1) {
|
|
if let Some(element) = pipeline.elements.get(0) {
|
|
return Err(ShellError::GenericError(
|
|
"error when loading nuon text".into(),
|
|
"could not load nuon text".into(),
|
|
Some(head),
|
|
None,
|
|
vec![ShellError::OutsideSpannedLabeledError(
|
|
string_input,
|
|
"error when loading".into(),
|
|
"excess values when loading".into(),
|
|
element.span(),
|
|
)],
|
|
));
|
|
} else {
|
|
return Err(ShellError::GenericError(
|
|
"error when loading nuon text".into(),
|
|
"could not load nuon text".into(),
|
|
Some(head),
|
|
None,
|
|
vec![ShellError::GenericError(
|
|
"error when loading".into(),
|
|
"excess values when loading".into(),
|
|
Some(head),
|
|
None,
|
|
Vec::new(),
|
|
)],
|
|
));
|
|
}
|
|
}
|
|
|
|
let expr = if block.pipelines.is_empty() {
|
|
Expression {
|
|
expr: Expr::Nothing,
|
|
span: head,
|
|
custom_completion: None,
|
|
ty: Type::Nothing,
|
|
}
|
|
} else {
|
|
let mut pipeline = block.pipelines.remove(0);
|
|
|
|
if let Some(expr) = pipeline.elements.get(1) {
|
|
return Err(ShellError::GenericError(
|
|
"error when loading nuon text".into(),
|
|
"could not load nuon text".into(),
|
|
Some(head),
|
|
None,
|
|
vec![ShellError::OutsideSpannedLabeledError(
|
|
string_input,
|
|
"error when loading".into(),
|
|
"detected a pipeline in nuon file".into(),
|
|
expr.span(),
|
|
)],
|
|
));
|
|
}
|
|
|
|
if pipeline.elements.is_empty() {
|
|
Expression {
|
|
expr: Expr::Nothing,
|
|
span: head,
|
|
custom_completion: None,
|
|
ty: Type::Nothing,
|
|
}
|
|
} else {
|
|
match pipeline.elements.remove(0) {
|
|
PipelineElement::Expression(_, expression)
|
|
| PipelineElement::Redirection(_, _, expression)
|
|
| PipelineElement::And(_, expression)
|
|
| PipelineElement::Or(_, expression)
|
|
| PipelineElement::SameTargetRedirection {
|
|
cmd: (_, expression),
|
|
..
|
|
}
|
|
| PipelineElement::SeparateRedirection {
|
|
out: (_, expression),
|
|
..
|
|
} => expression,
|
|
}
|
|
}
|
|
};
|
|
|
|
if let Some(err) = working_set.parse_errors.first() {
|
|
return Err(ShellError::GenericError(
|
|
"error when parsing nuon text".into(),
|
|
"could not parse nuon text".into(),
|
|
Some(head),
|
|
None,
|
|
vec![ShellError::OutsideSpannedLabeledError(
|
|
string_input,
|
|
"error when parsing".into(),
|
|
err.to_string(),
|
|
err.span(),
|
|
)],
|
|
));
|
|
}
|
|
|
|
let result = convert_to_value(expr, head, &string_input);
|
|
|
|
match result {
|
|
Ok(result) => Ok(result.into_pipeline_data_with_metadata(metadata)),
|
|
Err(err) => Err(ShellError::GenericError(
|
|
"error when loading nuon text".into(),
|
|
"could not load nuon text".into(),
|
|
Some(head),
|
|
None,
|
|
vec![err],
|
|
)),
|
|
}
|
|
}
|
|
}
|
|
|
|
fn convert_to_value(
|
|
expr: Expression,
|
|
span: Span,
|
|
original_text: &str,
|
|
) -> Result<Value, ShellError> {
|
|
match expr.expr {
|
|
Expr::BinaryOp(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"binary operators not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::UnaryNot(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"unary operators not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::Block(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"blocks not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::Closure(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"closures not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::Binary(val) => Ok(Value::Binary { val, span }),
|
|
Expr::Bool(val) => Ok(Value::Bool { val, span }),
|
|
Expr::Call(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"calls not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::CellPath(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"subexpressions and cellpaths not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::DateTime(dt) => Ok(Value::Date { val: dt, span }),
|
|
Expr::ExternalCall(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"calls not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::Filepath(val) => Ok(Value::String { val, span }),
|
|
Expr::Directory(val) => Ok(Value::String { val, span }),
|
|
Expr::Float(val) => Ok(Value::Float { val, span }),
|
|
Expr::FullCellPath(full_cell_path) => {
|
|
if !full_cell_path.tail.is_empty() {
|
|
Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"subexpressions and cellpaths not supported in nuon".into(),
|
|
expr.span,
|
|
))
|
|
} else {
|
|
convert_to_value(full_cell_path.head, span, original_text)
|
|
}
|
|
}
|
|
|
|
Expr::Garbage => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"extra tokens in input file".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::MatchPattern(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"extra tokens in input file".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::GlobPattern(val) => Ok(Value::String { val, span }),
|
|
Expr::ImportPattern(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"imports not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::Overlay(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"overlays not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::Int(val) => Ok(Value::Int { val, span }),
|
|
Expr::Keyword(kw, ..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
format!("{} not supported in nuon", String::from_utf8_lossy(&kw)),
|
|
expr.span,
|
|
)),
|
|
Expr::List(vals) => {
|
|
let mut output = vec![];
|
|
for val in vals {
|
|
output.push(convert_to_value(val, span, original_text)?);
|
|
}
|
|
|
|
Ok(Value::List { vals: output, span })
|
|
}
|
|
Expr::MatchBlock(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"match blocks not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::Nothing => Ok(Value::Nothing { span }),
|
|
Expr::Operator(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"operators not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::Range(from, next, to, operator) => {
|
|
let from = if let Some(f) = from {
|
|
convert_to_value(*f, span, original_text)?
|
|
} else {
|
|
Value::Nothing { span: expr.span }
|
|
};
|
|
|
|
let next = if let Some(s) = next {
|
|
convert_to_value(*s, span, original_text)?
|
|
} else {
|
|
Value::Nothing { span: expr.span }
|
|
};
|
|
|
|
let to = if let Some(t) = to {
|
|
convert_to_value(*t, span, original_text)?
|
|
} else {
|
|
Value::Nothing { span: expr.span }
|
|
};
|
|
|
|
Ok(Value::Range {
|
|
val: Box::new(Range::new(expr.span, from, next, to, &operator)?),
|
|
span: expr.span,
|
|
})
|
|
}
|
|
Expr::Record(key_vals) => {
|
|
let mut cols = vec![];
|
|
let mut vals = vec![];
|
|
|
|
for (key, val) in key_vals {
|
|
let key_str = match key.expr {
|
|
Expr::String(key_str) => key_str,
|
|
_ => {
|
|
return Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"only strings can be keys".into(),
|
|
key.span,
|
|
))
|
|
}
|
|
};
|
|
|
|
let value = convert_to_value(val, span, original_text)?;
|
|
|
|
cols.push(key_str);
|
|
vals.push(value);
|
|
}
|
|
|
|
Ok(Value::Record { cols, vals, span })
|
|
}
|
|
Expr::RowCondition(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"row conditions not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::Signature(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"signatures not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::String(s) => Ok(Value::String { val: s, span }),
|
|
Expr::StringInterpolation(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"string interpolation not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::Subexpression(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"subexpressions not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::Table(headers, cells) => {
|
|
let mut cols = vec![];
|
|
|
|
let mut output = vec![];
|
|
|
|
for key in headers {
|
|
let key_str = match key.expr {
|
|
Expr::String(key_str) => key_str,
|
|
_ => {
|
|
return Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"only strings can be keys".into(),
|
|
expr.span,
|
|
))
|
|
}
|
|
};
|
|
|
|
cols.push(key_str);
|
|
}
|
|
|
|
for row in cells {
|
|
let mut vals = vec![];
|
|
|
|
for cell in row {
|
|
vals.push(convert_to_value(cell, span, original_text)?);
|
|
}
|
|
|
|
if cols.len() != vals.len() {
|
|
return Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"table has mismatched columns".into(),
|
|
expr.span,
|
|
));
|
|
}
|
|
|
|
output.push(Value::Record {
|
|
cols: cols.clone(),
|
|
vals,
|
|
span,
|
|
});
|
|
}
|
|
|
|
Ok(Value::List { vals: output, span })
|
|
}
|
|
Expr::ValueWithUnit(val, unit) => {
|
|
let size = match val.expr {
|
|
Expr::Int(val) => val,
|
|
_ => {
|
|
return Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"non-integer unit value".into(),
|
|
expr.span,
|
|
))
|
|
}
|
|
};
|
|
|
|
match unit.item {
|
|
Unit::Byte => Ok(Value::Filesize { val: size, span }),
|
|
Unit::Kilobyte => Ok(Value::Filesize {
|
|
val: size * 1000,
|
|
span,
|
|
}),
|
|
Unit::Megabyte => Ok(Value::Filesize {
|
|
val: size * 1000 * 1000,
|
|
span,
|
|
}),
|
|
Unit::Gigabyte => Ok(Value::Filesize {
|
|
val: size * 1000 * 1000 * 1000,
|
|
span,
|
|
}),
|
|
Unit::Terabyte => Ok(Value::Filesize {
|
|
val: size * 1000 * 1000 * 1000 * 1000,
|
|
span,
|
|
}),
|
|
Unit::Petabyte => Ok(Value::Filesize {
|
|
val: size * 1000 * 1000 * 1000 * 1000 * 1000,
|
|
span,
|
|
}),
|
|
Unit::Exabyte => Ok(Value::Filesize {
|
|
val: size * 1000 * 1000 * 1000 * 1000 * 1000 * 1000,
|
|
span,
|
|
}),
|
|
Unit::Zettabyte => Ok(Value::Filesize {
|
|
val: size * 1000 * 1000 * 1000 * 1000 * 1000 * 1000 * 1000,
|
|
span,
|
|
}),
|
|
|
|
Unit::Kibibyte => Ok(Value::Filesize {
|
|
val: size * 1024,
|
|
span,
|
|
}),
|
|
Unit::Mebibyte => Ok(Value::Filesize {
|
|
val: size * 1024 * 1024,
|
|
span,
|
|
}),
|
|
Unit::Gibibyte => Ok(Value::Filesize {
|
|
val: size * 1024 * 1024 * 1024,
|
|
span,
|
|
}),
|
|
Unit::Tebibyte => Ok(Value::Filesize {
|
|
val: size * 1024 * 1024 * 1024 * 1024,
|
|
span,
|
|
}),
|
|
Unit::Pebibyte => Ok(Value::Filesize {
|
|
val: size * 1024 * 1024 * 1024 * 1024 * 1024,
|
|
span,
|
|
}),
|
|
Unit::Exbibyte => Ok(Value::Filesize {
|
|
val: size * 1024 * 1024 * 1024 * 1024 * 1024 * 1024,
|
|
span,
|
|
}),
|
|
Unit::Zebibyte => Ok(Value::Filesize {
|
|
val: size * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024,
|
|
span,
|
|
}),
|
|
|
|
Unit::Nanosecond => Ok(Value::Duration { val: size, span }),
|
|
Unit::Microsecond => Ok(Value::Duration {
|
|
val: size * 1000,
|
|
span,
|
|
}),
|
|
Unit::Millisecond => Ok(Value::Duration {
|
|
val: size * 1000 * 1000,
|
|
span,
|
|
}),
|
|
Unit::Second => Ok(Value::Duration {
|
|
val: size * 1000 * 1000 * 1000,
|
|
span,
|
|
}),
|
|
Unit::Minute => Ok(Value::Duration {
|
|
val: size * 1000 * 1000 * 1000 * 60,
|
|
span,
|
|
}),
|
|
Unit::Hour => Ok(Value::Duration {
|
|
val: size * 1000 * 1000 * 1000 * 60 * 60,
|
|
span,
|
|
}),
|
|
Unit::Day => match size.checked_mul(1000 * 1000 * 1000 * 60 * 60 * 24) {
|
|
Some(val) => Ok(Value::Duration { val, span }),
|
|
None => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"day duration too large".into(),
|
|
"day duration too large".into(),
|
|
expr.span,
|
|
)),
|
|
},
|
|
|
|
Unit::Week => match size.checked_mul(1000 * 1000 * 1000 * 60 * 60 * 24 * 7) {
|
|
Some(val) => Ok(Value::Duration { val, span }),
|
|
None => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"week duration too large".into(),
|
|
"week duration too large".into(),
|
|
expr.span,
|
|
)),
|
|
},
|
|
}
|
|
}
|
|
Expr::Var(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"variables not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
Expr::VarDecl(..) => Err(ShellError::OutsideSpannedLabeledError(
|
|
original_text.to_string(),
|
|
"Error when loading".into(),
|
|
"variable declarations not supported in nuon".into(),
|
|
expr.span,
|
|
)),
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod test {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_examples() {
|
|
use crate::test_examples;
|
|
|
|
test_examples(FromNuon {})
|
|
}
|
|
}
|