diff --git a/Cargo.toml b/Cargo.toml index 211d7bf482..9bcb9e8bef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,7 +59,6 @@ pretty_assertions = "1.0.0" [features] plugin = ["nu-plugin", "nu-parser/plugin", "nu-command/plugin", "nu-protocol/plugin", "nu-engine/plugin"] -fetch-support = ["nu-command/fetch"] default = [ "plugin", "inc", diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 0588982a1e..8ad34320d4 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -68,7 +68,7 @@ sha2 = "0.10.0" base64 = "0.13.0" encoding_rs = "0.8.30" num = { version = "0.4.0", optional = true } -reqwest = {version = "0.11", features = ["blocking"], optional = true } +reqwest = {version = "0.11", features = ["blocking"] } mime = "0.3.16" [target.'cfg(unix)'.dependencies] @@ -88,7 +88,6 @@ features = [ trash-support = ["trash"] plugin = ["nu-parser/plugin"] dataframe = ["polars", "num"] -fetch = ["reqwest"] [build-dependencies] shadow-rs = "0.8.1" diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index 70d2b862d8..a516e3044c 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -271,6 +271,7 @@ pub fn create_default_context(cwd: impl AsRef) -> EngineState { // Network bind_command! { + Fetch, Url, UrlHost, UrlPath, @@ -304,9 +305,6 @@ pub fn create_default_context(cwd: impl AsRef) -> EngineState { #[cfg(feature = "plugin")] bind_command!(Register); - #[cfg(feature = "fetch")] - bind_command!(Fetch); - // This is a WIP proof of concept // bind_command!(ListGitBranches, Git, GitCheckout, Source); diff --git a/crates/nu-command/src/network/fetch.rs b/crates/nu-command/src/network/fetch.rs index c67b882caa..4a5fc7c346 100644 --- a/crates/nu-command/src/network/fetch.rs +++ b/crates/nu-command/src/network/fetch.rs @@ -7,6 +7,7 @@ use nu_protocol::ByteStream; use nu_protocol::{ Category, Example, PipelineData, ShellError, Signature, Span, SyntaxShape, Value, }; +use reqwest::blocking::Response; use std::io::{BufRead, BufReader, Read}; @@ -162,98 +163,61 @@ fn helper( } match request.send() { - Ok(resp) => { - // let temp = std::fs::File::create("temp_dwl.txt")?; - // let mut b = BufWriter::new(temp); - // let _bytes = resp.copy_to(&mut b); - // let temp1 = std::fs::File::open("temp_dwl.txt")?; - // let a = BufReader::new(temp1); - - // TODO I guess we should check if all bytes were written/read... - match resp.headers().get("content-type") { - Some(content_type) => { - let content_type = content_type.to_str().map_err(|e| { - ShellError::LabeledError( - e.to_string(), - "MIME type were invalid".to_string(), - ) - })?; - let content_type = mime::Mime::from_str(content_type).map_err(|_| { - ShellError::LabeledError( - format!("MIME type unknown: {}", content_type), - "given unknown MIME type".to_string(), - ) - })?; - let ext = match (content_type.type_(), content_type.subtype()) { - (mime::TEXT, mime::PLAIN) => { - let path_extension = url::Url::parse(&requested_url) - .map_err(|_| { - ShellError::LabeledError( - format!("Cannot parse URL: {}", requested_url), - "cannot parse".to_string(), - ) - })? - .path_segments() - .and_then(|segments| segments.last()) - .and_then(|name| if name.is_empty() { None } else { Some(name) }) - .and_then(|name| { - PathBuf::from(name) - .extension() - .map(|name| name.to_string_lossy().to_string()) - }); - path_extension - } - _ => Some(content_type.subtype().to_string()), - }; - - let buffered_input = BufReader::new(resp); - - let output = PipelineData::ByteStream( - ByteStream { - stream: Box::new(BufferedReader { - input: buffered_input, - }), - ctrlc: engine_state.ctrlc.clone(), - }, - span, - None, - ); - - if raw { - return Ok(output); + Ok(resp) => match resp.headers().get("content-type") { + Some(content_type) => { + let content_type = content_type.to_str().map_err(|e| { + ShellError::LabeledError(e.to_string(), "MIME type were invalid".to_string()) + })?; + let content_type = mime::Mime::from_str(content_type).map_err(|_| { + ShellError::LabeledError( + format!("MIME type unknown: {}", content_type), + "given unknown MIME type".to_string(), + ) + })?; + let ext = match (content_type.type_(), content_type.subtype()) { + (mime::TEXT, mime::PLAIN) => { + let path_extension = url::Url::parse(&requested_url) + .map_err(|_| { + ShellError::LabeledError( + format!("Cannot parse URL: {}", requested_url), + "cannot parse".to_string(), + ) + })? + .path_segments() + .and_then(|segments| segments.last()) + .and_then(|name| if name.is_empty() { None } else { Some(name) }) + .and_then(|name| { + PathBuf::from(name) + .extension() + .map(|name| name.to_string_lossy().to_string()) + }); + path_extension } + _ => Some(content_type.subtype().to_string()), + }; - if let Some(ext) = ext { - match engine_state.find_decl(format!("from {}", ext).as_bytes()) { - Some(converter_id) => engine_state.get_decl(converter_id).run( - engine_state, - stack, - &Call::new(), - output, - ), - None => Ok(output), - } - } else { - Ok(output) - } + let output = response_to_buffer(resp, engine_state, span); + + if raw { + return Ok(output); } - None => { - let buffered_input = BufReader::new(resp); - let output = PipelineData::ByteStream( - ByteStream { - stream: Box::new(BufferedReader { - input: buffered_input, - }), - ctrlc: engine_state.ctrlc.clone(), - }, - span, - None, - ); + if let Some(ext) = ext { + match engine_state.find_decl(format!("from {}", ext).as_bytes()) { + Some(converter_id) => engine_state.get_decl(converter_id).run( + engine_state, + stack, + &Call::new(), + output, + ), + None => Ok(output), + } + } else { Ok(output) } } - } + None => Ok(response_to_buffer(resp, engine_state, span)), + }, Err(e) if e.is_timeout() => Err(ShellError::NetworkFailure( format!("Request to {} has timed out", requested_url), span, @@ -327,6 +291,25 @@ impl Iterator for BufferedReader { } } +fn response_to_buffer( + response: Response, + engine_state: &EngineState, + span: Span, +) -> nu_protocol::PipelineData { + let buffered_input = BufReader::new(response); + + PipelineData::ByteStream( + ByteStream { + stream: Box::new(BufferedReader { + input: buffered_input, + }), + ctrlc: engine_state.ctrlc.clone(), + }, + span, + None, + ) +} + // Only panics if the user agent is invalid but we define it statically so either // it always or never fails #[allow(clippy::unwrap_used)] diff --git a/crates/nu-command/src/network/mod.rs b/crates/nu-command/src/network/mod.rs index acf96c78bb..6f5eb9cf17 100644 --- a/crates/nu-command/src/network/mod.rs +++ b/crates/nu-command/src/network/mod.rs @@ -1,7 +1,5 @@ -#[cfg(feature = "fetch")] mod fetch; mod url; pub use self::url::*; -#[cfg(feature = "fetch")] pub use fetch::SubCommand as Fetch;