diff --git a/Cargo.lock b/Cargo.lock index 1c2a124d95..5a239233fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2685,6 +2685,7 @@ name = "nu-parser" version = "0.64.0" dependencies = [ "chrono", + "itertools", "log", "miette 4.7.1", "nu-path", diff --git a/crates/nu-parser/Cargo.toml b/crates/nu-parser/Cargo.toml index ea7ef2fa7e..e7264409ce 100644 --- a/crates/nu-parser/Cargo.toml +++ b/crates/nu-parser/Cargo.toml @@ -8,6 +8,7 @@ version = "0.64.0" [dependencies] chrono = "0.4.19" +itertools = "0.10" miette = "4.5.0" thiserror = "1.0.29" serde_json = "1.0" diff --git a/crates/nu-parser/src/parser.rs b/crates/nu-parser/src/parser.rs index f23e1f59e0..4394995cbf 100644 --- a/crates/nu-parser/src/parser.rs +++ b/crates/nu-parser/src/parser.rs @@ -22,6 +22,7 @@ use crate::parse_keywords::{ parse_use, }; +use itertools::Itertools; use log::trace; use std::{ collections::{HashMap, HashSet}, @@ -1206,9 +1207,13 @@ fn parse_binary_with_base( } fn decode_with_base(s: &str, base: u32, digits_per_byte: usize) -> Result, ParseIntError> { - (0..s.len()) - .step_by(digits_per_byte) - .map(|i| u8::from_str_radix(&s[i..i + digits_per_byte], base)) + s.chars() + .chunks(digits_per_byte) + .into_iter() + .map(|chunk| { + let str: String = chunk.collect(); + u8::from_str_radix(&str, base) + }) .collect() } diff --git a/crates/nu-parser/tests/test_parser.rs b/crates/nu-parser/tests/test_parser.rs index 155d4870e8..a09c11223b 100644 --- a/crates/nu-parser/tests/test_parser.rs +++ b/crates/nu-parser/tests/test_parser.rs @@ -159,6 +159,22 @@ pub fn parse_binary_with_invalid_octal_format() { assert!(!matches!(&expressions[0].expr, Expr::Binary(_))) } +#[test] +pub fn parse_binary_with_multi_byte_char() { + let engine_state = EngineState::new(); + let mut working_set = StateWorkingSet::new(&engine_state); + + // found using fuzzing, Rust can panic if you slice into this string + let contents = b"0x[\xEF\xBF\xBD]"; + let (block, err) = parse(&mut working_set, None, contents, true, &[]); + + assert!(err.is_none()); + assert!(block.len() == 1); + let expressions = &block[0]; + assert!(expressions.len() == 1); + assert!(!matches!(&expressions[0].expr, Expr::Binary(_))) +} + #[test] pub fn parse_string() { let engine_state = EngineState::new();