Add multipart/form-data uploads

This adds support for `multipart/form-data` (RFC 7578) uploads to
nushell.

Binary data is uploaded as files (`application/octet-stream`),
everything else is uploaded as plain text.
This commit is contained in:
Bruce Weirdan 2024-08-03 22:41:08 +02:00
parent 07e7c8c81f
commit 0f6b2b214a
No known key found for this signature in database
GPG Key ID: CFC3AAB181751B0D
6 changed files with 170 additions and 2 deletions

97
Cargo.lock generated
View File

@ -2908,6 +2908,7 @@ dependencies = [
"tango-bench", "tango-bench",
"tempfile", "tempfile",
"time", "time",
"ureq_multipart",
"winresource", "winresource",
] ]
@ -3121,6 +3122,7 @@ dependencies = [
"unicode-segmentation", "unicode-segmentation",
"unicode-width", "unicode-width",
"ureq", "ureq",
"ureq_multipart",
"url", "url",
"uu_cp", "uu_cp",
"uu_mkdir", "uu_mkdir",
@ -5104,6 +5106,21 @@ dependencies = [
"thiserror", "thiserror",
] ]
[[package]]
name = "ring"
version = "0.17.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
dependencies = [
"cc",
"cfg-if",
"getrandom",
"libc",
"spin",
"untrusted",
"windows-sys 0.52.0",
]
[[package]] [[package]]
name = "rkyv" name = "rkyv"
version = "0.7.44" version = "0.7.44"
@ -5314,6 +5331,38 @@ dependencies = [
"windows-sys 0.52.0", "windows-sys 0.52.0",
] ]
[[package]]
name = "rustls"
version = "0.23.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044"
dependencies = [
"log",
"once_cell",
"ring",
"rustls-pki-types",
"rustls-webpki",
"subtle",
"zeroize",
]
[[package]]
name = "rustls-pki-types"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d"
[[package]]
name = "rustls-webpki"
version = "0.102.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e"
dependencies = [
"ring",
"rustls-pki-types",
"untrusted",
]
[[package]] [[package]]
name = "rustversion" name = "rustversion"
version = "1.0.15" version = "1.0.15"
@ -5737,6 +5786,12 @@ dependencies = [
"windows-sys 0.52.0", "windows-sys 0.52.0",
] ]
[[package]]
name = "spin"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
[[package]] [[package]]
name = "sqlparser" name = "sqlparser"
version = "0.47.0" version = "0.47.0"
@ -5871,6 +5926,12 @@ dependencies = [
"syn 2.0.60", "syn 2.0.60",
] ]
[[package]]
name = "subtle"
version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]] [[package]]
name = "supports-color" name = "supports-color"
version = "2.1.0" version = "2.1.0"
@ -6482,6 +6543,12 @@ version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
[[package]]
name = "untrusted"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]] [[package]]
name = "ureq" name = "ureq"
version = "2.10.0" version = "2.10.0"
@ -6494,9 +6561,24 @@ dependencies = [
"log", "log",
"native-tls", "native-tls",
"once_cell", "once_cell",
"rustls",
"rustls-pki-types",
"serde", "serde",
"serde_json", "serde_json",
"url", "url",
"webpki-roots",
]
[[package]]
name = "ureq_multipart"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22baf2d124865fc4d505f5942222a57f6a3eae8a133819d7fd6194423e3f6e91"
dependencies = [
"mime",
"mime_guess",
"rand",
"ureq",
] ]
[[package]] [[package]]
@ -6902,6 +6984,15 @@ dependencies = [
"url", "url",
] ]
[[package]]
name = "webpki-roots"
version = "0.26.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd"
dependencies = [
"rustls-pki-types",
]
[[package]] [[package]]
name = "which" name = "which"
version = "6.0.1" version = "6.0.1"
@ -7357,6 +7448,12 @@ dependencies = [
"syn 2.0.60", "syn 2.0.60",
] ]
[[package]]
name = "zeroize"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
[[package]] [[package]]
name = "zip" name = "zip"
version = "0.6.6" version = "0.6.6"

View File

@ -166,6 +166,7 @@ umask = "2.1"
unicode-segmentation = "1.11" unicode-segmentation = "1.11"
unicode-width = "0.1" unicode-width = "0.1"
ureq = { version = "2.10", default-features = false } ureq = { version = "2.10", default-features = false }
ureq_multipart = "1.1.1"
url = "2.2" url = "2.2"
uu_cp = "0.0.27" uu_cp = "0.0.27"
uu_mkdir = "0.0.27" uu_mkdir = "0.0.27"
@ -210,6 +211,7 @@ mimalloc = { version = "0.1.42", default-features = false, optional = true }
serde_json = { workspace = true } serde_json = { workspace = true }
simplelog = "0.12" simplelog = "0.12"
time = "0.3" time = "0.3"
ureq_multipart = { workspace = true }
[target.'cfg(not(target_os = "windows"))'.dependencies] [target.'cfg(not(target_os = "windows"))'.dependencies]
# Our dependencies don't use OpenSSL on Windows # Our dependencies don't use OpenSSL on Windows

View File

@ -90,6 +90,7 @@ titlecase = { workspace = true }
toml = { workspace = true, features = ["preserve_order"]} toml = { workspace = true, features = ["preserve_order"]}
unicode-segmentation = { workspace = true } unicode-segmentation = { workspace = true }
ureq = { workspace = true, default-features = false, features = ["charset", "gzip", "json", "native-tls"] } ureq = { workspace = true, default-features = false, features = ["charset", "gzip", "json", "native-tls"] }
ureq_multipart = { workspace = true }
url = { workspace = true } url = { workspace = true }
uu_cp = { workspace = true } uu_cp = { workspace = true }
uu_mkdir = { workspace = true } uu_mkdir = { workspace = true }
@ -146,4 +147,4 @@ quickcheck = { workspace = true }
quickcheck_macros = { workspace = true } quickcheck_macros = { workspace = true }
rstest = { workspace = true, default-features = false } rstest = { workspace = true, default-features = false }
pretty_assertions = { workspace = true } pretty_assertions = { workspace = true }
tempfile = { workspace = true } tempfile = { workspace = true }

View File

@ -8,18 +8,21 @@ use nu_engine::command_prelude::*;
use nu_protocol::{ByteStream, Signals}; use nu_protocol::{ByteStream, Signals};
use std::{ use std::{
collections::HashMap, collections::HashMap,
io::Cursor,
path::PathBuf, path::PathBuf,
str::FromStr, str::FromStr,
sync::mpsc::{self, RecvTimeoutError}, sync::mpsc::{self, RecvTimeoutError},
time::Duration, time::Duration,
}; };
use ureq::{Error, ErrorKind, Request, Response}; use ureq::{Error, ErrorKind, Request, Response};
use ureq_multipart::MultipartBuilder;
use url::Url; use url::Url;
#[derive(PartialEq, Eq)] #[derive(PartialEq, Eq)]
pub enum BodyType { pub enum BodyType {
Json, Json,
Form, Form,
Multipart,
Unknown, Unknown,
} }
@ -210,6 +213,7 @@ pub fn send_request(
let (body_type, req) = match content_type { let (body_type, req) = match content_type {
Some(it) if it == "application/json" => (BodyType::Json, request), Some(it) if it == "application/json" => (BodyType::Json, request),
Some(it) if it == "application/x-www-form-urlencoded" => (BodyType::Form, request), Some(it) if it == "application/x-www-form-urlencoded" => (BodyType::Form, request),
Some(it) if it == "multipart/form-data" => (BodyType::Multipart, request),
Some(it) => { Some(it) => {
let r = request.clone().set("Content-Type", &it); let r = request.clone().set("Content-Type", &it);
(BodyType::Unknown, r) (BodyType::Unknown, r)
@ -265,6 +269,35 @@ pub fn send_request(
}; };
send_cancellable_request(&request_url, Box::new(request_fn), span, signals) send_cancellable_request(&request_url, Box::new(request_fn), span, signals)
} }
// multipart form upload
Value::Record { val, .. } if body_type == BodyType::Multipart => {
let mut builder = MultipartBuilder::new();
let err = |e| {
ShellErrorOrRequestError::ShellError(ShellError::IOError {
msg: format!("failed to build multipart data: {}", e),
})
};
for (col, val) in val.into_owned() {
if let Value::Binary { val, .. } = val {
builder = builder
.add_stream(&mut Cursor::new(val), &col, Some(&col), None)
.map_err(err)?;
} else {
builder = builder
.add_text(&col, &val.coerce_into_string()?)
.map_err(err)?;
}
}
let (content_type, data) = builder.finish().map_err(err)?;
let request_fn =
move || req.set("Content-Type", &content_type).send_bytes(&data);
send_cancellable_request(&request_url, Box::new(request_fn), span, signals)
}
Value::List { vals, .. } if body_type == BodyType::Form => { Value::List { vals, .. } if body_type == BodyType::Form => {
if vals.len() % 2 != 0 { if vals.len() % 2 != 0 {
return Err(ShellErrorOrRequestError::ShellError(ShellError::IOError { return Err(ShellErrorOrRequestError::ShellError(ShellError::IOError {

View File

@ -127,6 +127,11 @@ impl Command for SubCommand {
example: "open foo.json | http post https://www.example.com", example: "open foo.json | http post https://www.example.com",
result: None, result: None,
}, },
Example {
description: "Upload a file to example.com",
example: "http post --content-type multipart/form-data https://www.example.com { audio: (open -r file.mp3) }",
result: None,
},
] ]
} }
} }

View File

@ -1,4 +1,4 @@
use mockito::Server; use mockito::{Matcher, Server, ServerOpts};
use nu_test_support::{nu, pipeline}; use nu_test_support::{nu, pipeline};
#[test] #[test]
@ -197,3 +197,33 @@ fn http_post_redirect_mode_error() {
"Redirect encountered when redirect handling mode was 'error' (301 Moved Permanently)" "Redirect encountered when redirect handling mode was 'error' (301 Moved Permanently)"
)); ));
} }
#[test]
fn http_post_multipart_is_success() {
let mut server = Server::new_with_opts(ServerOpts {
assert_on_drop: true,
..Default::default()
});
let _mock = server
.mock("POST", "/")
.match_header(
"content-type",
Matcher::Regex("multipart/form-data; boundary=.*".to_string()),
)
.match_body(Matcher::AllOf(vec![
Matcher::Regex(r#"Content-Disposition: form-data; name="foo""#.to_string()),
Matcher::Regex(r#"Content-Type: application/octet-stream"#.to_string()),
Matcher::Regex(r#"bar"#.to_string()),
]))
.with_status(200)
.create();
let actual = nu!(pipeline(
format!(
"http post --content-type multipart/form-data {url} {{foo: ('bar' | into binary) }}",
url = server.url()
)
.as_str()
));
assert!(actual.out.is_empty())
}