From 0f7c9c7f72041b9402970d2db0a54701488312e8 Mon Sep 17 00:00:00 2001 From: Jack Wright Date: Fri, 2 Aug 2024 11:22:47 -0700 Subject: [PATCH 1/5] Attempt to guess the content type of a file when opening with --raw and set it in the pipeline metadata --- crates/nu-command/src/filesystem/open.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/nu-command/src/filesystem/open.rs b/crates/nu-command/src/filesystem/open.rs index 0351d1d9b2..4b47169b22 100644 --- a/crates/nu-command/src/filesystem/open.rs +++ b/crates/nu-command/src/filesystem/open.rs @@ -146,11 +146,19 @@ impl Command for Open { } }; + let content_type = if raw { + mime_guess::from_path(path) + .first() + .map(|mime| mime.to_string()) + } else { + None + }; + let stream = PipelineData::ByteStream( ByteStream::file(file, call_span, engine_state.signals().clone()), Some(PipelineMetadata { data_source: DataSource::FilePath(path.to_path_buf()), - content_type: None, + content_type, }), ); From 50ecdbbe15e509abd15ee6ab34a6113e6fa3108c Mon Sep 17 00:00:00 2001 From: Jack Wright Date: Mon, 5 Aug 2024 13:53:41 -0700 Subject: [PATCH 2/5] content-type testing --- crates/nu-command/src/filesystem/open.rs | 25 ++++++++++++++++++++--- crates/nu-command/src/formats/to/toml.rs | 21 ++++++++++++++----- crates/nu-command/src/formats/to/yaml.rs | 1 + crates/nu-command/tests/commands/open.rs | 26 ++++++++++++++++++++++++ crates/nu-test-support/src/lib.rs | 1 + 5 files changed, 66 insertions(+), 8 deletions(-) diff --git a/crates/nu-command/src/filesystem/open.rs b/crates/nu-command/src/filesystem/open.rs index 4b47169b22..23e95d1aed 100644 --- a/crates/nu-command/src/filesystem/open.rs +++ b/crates/nu-command/src/filesystem/open.rs @@ -147,9 +147,9 @@ impl Command for Open { }; let content_type = if raw { - mime_guess::from_path(path) - .first() - .map(|mime| mime.to_string()) + path.extension() + .map(|ext| ext.to_string_lossy().to_string()) + .and_then(|ref s| detect_content_type(s)) } else { None }; @@ -276,3 +276,22 @@ fn extract_extensions(filename: &str) -> Vec { extensions } + +fn detect_content_type(extension: &str) -> Option { + // This will allow the overriding of metadata to be consistent with + // the content type + match extension { + // Per RFC-9512, application/yaml should be used + "yaml" | "yml" => Some("application/yaml".to_string()), + _ => mime_guess::from_ext(extension) + .first() + .map(|mime| mime.to_string()), + } +} + +#[cfg(test)] +mod test { + + #[test] + fn test_content_type() {} +} diff --git a/crates/nu-command/src/formats/to/toml.rs b/crates/nu-command/src/formats/to/toml.rs index ed1490231c..c70864b7f8 100644 --- a/crates/nu-command/src/formats/to/toml.rs +++ b/crates/nu-command/src/formats/to/toml.rs @@ -1,6 +1,6 @@ use chrono::{DateTime, Datelike, FixedOffset, Timelike}; use nu_engine::command_prelude::*; -use nu_protocol::ast::PathMember; +use nu_protocol::{ast::PathMember, PipelineMetadata}; #[derive(Clone)] pub struct ToToml; @@ -100,9 +100,18 @@ fn toml_into_pipeline_data( toml_value: &toml::Value, value_type: Type, span: Span, + metadata: Option, ) -> Result { + let new_md = Some( + metadata + .unwrap_or_default() + .with_content_type(Some("text/x-toml".into())), + ); + match toml::to_string_pretty(&toml_value) { - Ok(serde_toml_string) => Ok(Value::string(serde_toml_string, span).into_pipeline_data()), + Ok(serde_toml_string) => { + Ok(Value::string(serde_toml_string, span).into_pipeline_data_with_metadata(new_md)) + } _ => Ok(Value::error( ShellError::CantConvert { to_type: "TOML".into(), @@ -112,7 +121,7 @@ fn toml_into_pipeline_data( }, span, ) - .into_pipeline_data()), + .into_pipeline_data_with_metadata(new_md)), } } @@ -139,6 +148,7 @@ fn to_toml( input: PipelineData, span: Span, ) -> Result { + let metadata = input.metadata(); let value = input.into_value(span)?; let toml_value = value_to_toml_value(engine_state, &value, span)?; @@ -148,10 +158,11 @@ fn to_toml( vec.iter().next().expect("this should never trigger"), value.get_type(), span, + metadata, ), - _ => toml_into_pipeline_data(&toml_value, value.get_type(), span), + _ => toml_into_pipeline_data(&toml_value, value.get_type(), span, metadata), }, - _ => toml_into_pipeline_data(&toml_value, value.get_type(), span), + _ => toml_into_pipeline_data(&toml_value, value.get_type(), span, metadata), } } diff --git a/crates/nu-command/src/formats/to/yaml.rs b/crates/nu-command/src/formats/to/yaml.rs index c1693cda7d..ae923c1fc8 100644 --- a/crates/nu-command/src/formats/to/yaml.rs +++ b/crates/nu-command/src/formats/to/yaml.rs @@ -98,6 +98,7 @@ fn to_yaml(input: PipelineData, head: Span) -> Result let metadata = input .metadata() .unwrap_or_default() + // Per RFC-9512, application/yaml should be used .with_content_type(Some("application/yaml".into())); let value = input.into_value(head)?; diff --git a/crates/nu-command/tests/commands/open.rs b/crates/nu-command/tests/commands/open.rs index ac5e2f99e5..3ca8d2650a 100644 --- a/crates/nu-command/tests/commands/open.rs +++ b/crates/nu-command/tests/commands/open.rs @@ -379,3 +379,29 @@ fn open_files_inside_glob_metachars_dir() { assert!(actual.out.contains("hello")); }); } + +#[test] +fn test_content_types_with_open_raw() { + Playground::setup("open_files_content_type_test", |dirs, _| { + let result = nu!(cwd: dirs.formats(), "open --raw random_numbers.csv | metadata"); + assert!(result.out.contains("text/csv")); + let result = nu!(cwd: dirs.formats(), "open --raw caco3_plastics.tsv | metadata"); + assert!(result.out.contains("text/tab-separated-values")); + let result = nu!(cwd: dirs.formats(), "open --raw sample-simple.json | metadata"); + assert!(result.out.contains("application/json")); + let result = nu!(cwd: dirs.formats(), "open --raw sample.ini | metadata"); + assert!(result.out.contains("text/plain")); + let result = nu!(cwd: dirs.formats(), "open --raw sample_data.xlsx | metadata"); + assert!(result + .out + .contains("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")); + let result = nu!(cwd: dirs.formats(), "open --raw sample_def.nu | metadata"); + assert!(!result.out.contains("content_type")); + let result = nu!(cwd: dirs.formats(), "open --raw sample.eml | metadata"); + assert!(result.out.contains("message/rfc822")); + let result = nu!(cwd: dirs.formats(), "open --raw cargo_sample.toml | metadata"); + assert!(result.out.contains("text/x-toml")); + let result = nu!(cwd: dirs.formats(), "open --raw appveyor.yml | metadata"); + assert!(result.out.contains("application/yaml")); + }) +} diff --git a/crates/nu-test-support/src/lib.rs b/crates/nu-test-support/src/lib.rs index c6cadbcbd8..930e3792dc 100644 --- a/crates/nu-test-support/src/lib.rs +++ b/crates/nu-test-support/src/lib.rs @@ -9,6 +9,7 @@ use std::process::ExitStatus; // Needs to be reexported for `nu!` macro pub use nu_path; +#[derive(Debug)] pub struct Outcome { pub out: String, pub err: String, From 6a90da56d53273ac3038f4d7a3c8b5a69d633459 Mon Sep 17 00:00:00 2001 From: Jack Wright Date: Mon, 5 Aug 2024 14:00:02 -0700 Subject: [PATCH 3/5] removed the xlsx case --- crates/nu-command/tests/commands/open.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/nu-command/tests/commands/open.rs b/crates/nu-command/tests/commands/open.rs index 3ca8d2650a..18e4dc5b8b 100644 --- a/crates/nu-command/tests/commands/open.rs +++ b/crates/nu-command/tests/commands/open.rs @@ -391,10 +391,6 @@ fn test_content_types_with_open_raw() { assert!(result.out.contains("application/json")); let result = nu!(cwd: dirs.formats(), "open --raw sample.ini | metadata"); assert!(result.out.contains("text/plain")); - let result = nu!(cwd: dirs.formats(), "open --raw sample_data.xlsx | metadata"); - assert!(result - .out - .contains("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")); let result = nu!(cwd: dirs.formats(), "open --raw sample_def.nu | metadata"); assert!(!result.out.contains("content_type")); let result = nu!(cwd: dirs.formats(), "open --raw sample.eml | metadata"); From 23aa1ad22ea6ede9af2a0a5dba8818bd7bf856f6 Mon Sep 17 00:00:00 2001 From: Jack Wright Date: Mon, 5 Aug 2024 14:06:35 -0700 Subject: [PATCH 4/5] tweaked xlsx test case --- crates/nu-command/tests/commands/open.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/nu-command/tests/commands/open.rs b/crates/nu-command/tests/commands/open.rs index 18e4dc5b8b..f8153bec39 100644 --- a/crates/nu-command/tests/commands/open.rs +++ b/crates/nu-command/tests/commands/open.rs @@ -391,6 +391,10 @@ fn test_content_types_with_open_raw() { assert!(result.out.contains("application/json")); let result = nu!(cwd: dirs.formats(), "open --raw sample.ini | metadata"); assert!(result.out.contains("text/plain")); + let result = nu!(cwd: dirs.formats(), "open --raw sample_data.xlsx | metadata"); + assert!(result + .out + .contains("vnd.openxmlformats-officedocument")); let result = nu!(cwd: dirs.formats(), "open --raw sample_def.nu | metadata"); assert!(!result.out.contains("content_type")); let result = nu!(cwd: dirs.formats(), "open --raw sample.eml | metadata"); From e73c1bf26d6bb37585010205f1b72feae8cc4382 Mon Sep 17 00:00:00 2001 From: Jack Wright Date: Mon, 5 Aug 2024 14:10:46 -0700 Subject: [PATCH 5/5] formatting --- crates/nu-command/tests/commands/open.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/nu-command/tests/commands/open.rs b/crates/nu-command/tests/commands/open.rs index f8153bec39..b50c6c9e6c 100644 --- a/crates/nu-command/tests/commands/open.rs +++ b/crates/nu-command/tests/commands/open.rs @@ -392,9 +392,7 @@ fn test_content_types_with_open_raw() { let result = nu!(cwd: dirs.formats(), "open --raw sample.ini | metadata"); assert!(result.out.contains("text/plain")); let result = nu!(cwd: dirs.formats(), "open --raw sample_data.xlsx | metadata"); - assert!(result - .out - .contains("vnd.openxmlformats-officedocument")); + assert!(result.out.contains("vnd.openxmlformats-officedocument")); let result = nu!(cwd: dirs.formats(), "open --raw sample_def.nu | metadata"); assert!(!result.out.contains("content_type")); let result = nu!(cwd: dirs.formats(), "open --raw sample.eml | metadata");