From c259ef41bd8bd7cbc57e664e82667e100d495671 Mon Sep 17 00:00:00 2001 From: WindSoilder Date: Thu, 10 Nov 2022 06:07:38 +0800 Subject: [PATCH] update polar to 0.25 (#6988) --- Cargo.lock | 311 +++++++++++------- crates/nu-command/Cargo.toml | 2 +- .../src/dataframe/eager/describe.rs | 2 +- crates/nu-command/src/dataframe/eager/open.rs | 1 + .../src/dataframe/eager/sql_context.rs | 8 +- .../src/dataframe/eager/sql_expr.rs | 2 +- .../expressions/expressions_macro.rs | 65 +++- .../src/dataframe/lazy/aggregate.rs | 17 +- .../src/dataframe/lazy/macro_commands.rs | 56 +++- .../dataframe/series/indexes/set_with_idx.rs | 2 +- .../src/dataframe/values/nu_expression/mod.rs | 83 +---- .../src/dataframe/values/nu_when/mod.rs | 6 +- 12 files changed, 327 insertions(+), 228 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 651c6cb421..ba2a4e7ad5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "ahash" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "464b3811b747f8f7ebc8849c9c728c39f6ac98a055edad93baf9eb330e3f8f9d" +dependencies = [ + "cfg-if 1.0.0", + "getrandom 0.2.7", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.19" @@ -123,17 +135,18 @@ dependencies = [ [[package]] name = "arrow2" -version = "0.13.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc54f0b14083abaf6bc71cf1aeccd7831a24b1e29d07683ba9a4a0f6c5d9326" +checksum = "ee6f62e41078c967a4c063fcbdfd3801a2a9632276402c045311c4d73d0845f3" dependencies = [ - "ahash", + "ahash 0.7.6", "arrow-format", "base64", "bytemuck", "chrono", "dyn-clone", "either", + "ethnum", "fallible-streaming-iterator", "foreign_vec", "futures", @@ -305,15 +318,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "bitpacking" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8c7d2ac73c167c06af4a5f37e6e59d84148d57ccbe4480b76f0273eefea82d7" -dependencies = [ - "crunchy", -] - [[package]] name = "block-buffer" version = "0.10.3" @@ -556,13 +560,13 @@ dependencies = [ [[package]] name = "comfy-table" -version = "5.0.1" +version = "6.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b103d85ca6e209388771bfb7aa6b68a7aeec4afbf6f0a0264bfbf50360e5212e" +checksum = "1090f39f45786ec6dc6286f8ea9c75d0a7ef0a0d3cda674cef0c3af7b307fbc2" dependencies = [ - "crossterm 0.23.2", - "strum 0.23.0", - "strum_macros 0.23.1", + "crossterm 0.25.0", + "strum", + "strum_macros", "unicode-width", ] @@ -743,22 +747,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "crossterm" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2102ea4f781910f8a5b98dd061f4c2023f479ce7bb1236330099ceb5a93cf17" -dependencies = [ - "bitflags", - "crossterm_winapi", - "libc", - "mio 0.8.4", - "parking_lot", - "signal-hook", - "signal-hook-mio", - "winapi 0.3.9", -] - [[package]] name = "crossterm" version = "0.24.0" @@ -776,6 +764,22 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "crossterm" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" +dependencies = [ + "bitflags", + "crossterm_winapi", + "libc", + "mio 0.8.4", + "parking_lot", + "signal-hook", + "signal-hook-mio", + "winapi 0.3.9", +] + [[package]] name = "crossterm_winapi" version = "0.9.0" @@ -785,12 +789,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - [[package]] name = "crypto-common" version = "0.1.6" @@ -1059,6 +1057,18 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "enum_dispatch" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eb359f1476bf611266ac1f5355bc14aeca37b299d0ebccc038ee7058891c9cb" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "env_logger" version = "0.8.4" @@ -1099,6 +1109,12 @@ dependencies = [ "libc", ] +[[package]] +name = "ethnum" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eac3c0b9fa6eb75255ebb42c0ba3e2210d102a66d2795afef6fed668f373311" + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1172,6 +1188,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1409,8 +1434,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ "cfg-if 1.0.0", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -1488,6 +1515,17 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +[[package]] +name = "halfbrown" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce69ed202df415a3d4a01e6f3341320ca88b9bd4f0bf37be6fa239cdea06d9bf" +dependencies = [ + "fxhash", + "hashbrown", + "serde", +] + [[package]] name = "hamcrest2" version = "0.3.0" @@ -1519,7 +1557,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash", + "ahash 0.7.6", "rayon", ] @@ -1545,15 +1583,6 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.4.0" @@ -1744,16 +1773,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" -dependencies = [ - "async-trait", - "futures-util", -] - [[package]] name = "inventory" version = "0.2.3" @@ -1863,9 +1882,12 @@ dependencies = [ [[package]] name = "json-deserializer" -version = "0.3.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47631885425c482fcf2dc4b182fc973c3c5b81a8f43a028055559bd24cccfa6e" +checksum = "d784d2d481d0bace3450572391d6076dd6d10c66c0ebc1a0be037b3b420664bd" +dependencies = [ + "indexmap", +] [[package]] name = "json_to_table" @@ -3157,29 +3179,28 @@ dependencies = [ ] [[package]] -name = "parquet-format-async-temp" -version = "0.3.1" +name = "parquet-format-safe" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1a672c84c3e5b5eb6530286b2d22cc1ea8e1e3560e4c314218d6ab749c6db99" +checksum = "1131c54b167dd4e4799ce762e1ab01549ebb94d5bdd13e6ec1b467491c378e1f" dependencies = [ "async-trait", "futures", - "integer-encoding", ] [[package]] name = "parquet2" -version = "0.14.3" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa7afeef767fb16a37216cdf906268ae10db9825aa1ed8723b8493b1e99b0b" +checksum = "7752b1a7d61b278f36820ba05557a2a8bfcb17f3559254577ef447beda0c4975" dependencies = [ "async-stream", - "bitpacking", "brotli", "flate2", "futures", "lz4", - "parquet-format-async-temp", + "parquet-format-safe", + "seq-macro", "snap", "streaming-decompression", "zstd", @@ -3422,9 +3443,9 @@ dependencies = [ [[package]] name = "polars" -version = "0.23.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75b1077fda63c0f67acc1cdc8586e7afce419be1e85bf7dfa8935e0e266d6b3" +checksum = "112b33c016ec5949e414016b24d329a2188051e4f65d59a555986ecd7c7387ae" dependencies = [ "polars-core", "polars-io", @@ -3435,9 +3456,9 @@ dependencies = [ [[package]] name = "polars-arrow" -version = "0.23.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7b28f858b252436550679609a23be34d62705faf783887f172f845eb58bcb8b" +checksum = "7aac9a1d70c683cda8dd4958bd489d8c0206c5ab1435a656a14563c6415f5a64" dependencies = [ "arrow2", "hashbrown", @@ -3448,11 +3469,11 @@ dependencies = [ [[package]] name = "polars-core" -version = "0.23.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeaec1ca3ac4829ca24b33743adeeb323a43b5a85515bfce20c2c81799c82790" +checksum = "fda4c06062d5e22036e686762bd2da2cf2fc003728e123a99f9c2df7335afc10" dependencies = [ - "ahash", + "ahash 0.8.1", "anyhow", "arrow2", "bitflags", @@ -3476,11 +3497,11 @@ dependencies = [ [[package]] name = "polars-io" -version = "0.23.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51405e46f93e306a3c9280c60ba1101c662e8a6dab33344680d31c3161045f1c" +checksum = "125fd8302050f3b6be46f7a9a92a4ac62dcab738c4e32cd2af7b9be5f4cdf709" dependencies = [ - "ahash", + "ahash 0.8.1", "anyhow", "arrow2", "csv-core", @@ -3498,20 +3519,65 @@ dependencies = [ "rayon", "regex", "serde", - "serde_json", + "simd-json", "simdutf8", ] [[package]] name = "polars-lazy" -version = "0.23.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1340af778bc8124180d8ca1a566f076a5339566a207a42130796048b087fe977" +checksum = "55376a76c7bd050bb42fd124c290e1908e7ebf8f4c6cce6c4c41e56402f10a33" dependencies = [ - "ahash", + "ahash 0.8.1", "bitflags", "glob", - "parking_lot", + "polars-arrow", + "polars-core", + "polars-io", + "polars-ops", + "polars-pipe", + "polars-plan", + "polars-time", + "polars-utils", + "rayon", +] + +[[package]] +name = "polars-ops" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfd3f6552b3e9539634c35047f372db331b6227f75c36fcbe4670ab58bbcbeb3" +dependencies = [ + "arrow2", + "polars-arrow", + "polars-core", + "polars-utils", +] + +[[package]] +name = "polars-pipe" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa865a4fc6dcfb9967505c4714c29277898e21dd29ea7633a3f9d1abbe879a7d" +dependencies = [ + "enum_dispatch", + "hashbrown", + "num 0.4.0", + "polars-core", + "polars-io", + "polars-plan", + "polars-utils", + "rayon", +] + +[[package]] +name = "polars-plan" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673480b0ee55b0abb5be247862bfe1f3b623897f504006a10adbc25c878bf531" +dependencies = [ + "ahash 0.8.1", "polars-arrow", "polars-core", "polars-io", @@ -3522,21 +3588,11 @@ dependencies = [ "serde", ] -[[package]] -name = "polars-ops" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1812e5d5e589d5bd23f8d89dcd8bd4508082c50d055b8ff5fafb6f2a519c9a" -dependencies = [ - "polars-arrow", - "polars-core", -] - [[package]] name = "polars-time" -version = "0.23.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc4ebe97d601a4b443337df71d0b7e673fce953654871c3311850ea394d48297" +checksum = "783e53abda73c226807e850d91de67ab376d91866f378306d3e7f6b9844c17de" dependencies = [ "chrono", "lexical", @@ -3548,11 +3604,10 @@ dependencies = [ [[package]] name = "polars-utils" -version = "0.23.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea836afadcddee3f1a513dae7624f6d7d0d64abb129063ec7476b8347c8725b" +checksum = "f2d5b6ec3c45989c1fbaf0e13a3832f4b5e418e776518d7a24d62672e5366364" dependencies = [ - "parking_lot", "rayon", ] @@ -3970,8 +4025,8 @@ dependencies = [ "serde", "serde_json", "strip-ansi-escapes", - "strum 0.24.1", - "strum_macros 0.24.3", + "strum", + "strum_macros", "thiserror", "unicode-segmentation", "unicode-width", @@ -4370,6 +4425,12 @@ dependencies = [ "pest", ] +[[package]] +name = "seq-macro" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0772c5c30e1a0d91f6834f8e545c69281c099dfa9a3ac58d96a9fd629c8d4898" + [[package]] name = "serde" version = "1.0.145" @@ -4541,6 +4602,19 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-json" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd78b840b9de64fa3f7d72909b76343849f68e8c3d32608db8d38e4e5481f84" +dependencies = [ + "halfbrown", + "serde", + "serde_json", + "simdutf8", + "value-trait", +] + [[package]] name = "simdutf8" version = "0.1.4" @@ -4699,38 +4773,19 @@ dependencies = [ "vte", ] -[[package]] -name = "strum" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" - [[package]] name = "strum" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" -[[package]] -name = "strum_macros" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" -dependencies = [ - "heck 0.3.3", - "proc-macro2", - "quote", - "rustversion", - "syn", -] - [[package]] name = "strum_macros" version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro2", "quote", "rustversion", @@ -4845,7 +4900,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "beca1b4eaceb4f2755df858b88d9b9315b7ccfd1ffd0d7a48a52602301f01a57" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro-error", "proc-macro2", "quote", @@ -5310,6 +5365,18 @@ dependencies = [ "getrandom 0.2.7", ] +[[package]] +name = "value-trait" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0a635407649b66e125e4d2ffd208153210179f8c7c8b71c030aa2ad3eeb4c8f" +dependencies = [ + "float-cmp", + "halfbrown", + "itoa 1.0.4", + "ryu", +] + [[package]] name = "vcell" version = "0.1.3" diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 04bb746786..cf0edbf35c 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -104,7 +104,7 @@ version = "2.1.3" optional = true [dependencies.polars] -version = "0.23.2" +version = "0.25.0" optional = true features = [ "arg_where", diff --git a/crates/nu-command/src/dataframe/eager/describe.rs b/crates/nu-command/src/dataframe/eager/describe.rs index 5f521af163..2af4a6f6ff 100644 --- a/crates/nu-command/src/dataframe/eager/describe.rs +++ b/crates/nu-command/src/dataframe/eager/describe.rs @@ -199,7 +199,7 @@ fn command( _ => None, }; - let std = match col.std_as_series().get(0) { + let std = match col.std_as_series(0).get(0) { AnyValue::Float64(v) => Some(v), _ => None, }; diff --git a/crates/nu-command/src/dataframe/eager/open.rs b/crates/nu-command/src/dataframe/eager/open.rs index 4c9d4e2860..7a315c84d2 100644 --- a/crates/nu-command/src/dataframe/eager/open.rs +++ b/crates/nu-command/src/dataframe/eager/open.rs @@ -213,6 +213,7 @@ fn from_ipc( cache: true, rechunk: false, row_count: None, + memmap: true, }; let df: NuLazyFrame = LazyFrame::scan_ipc(file, args) diff --git a/crates/nu-command/src/dataframe/eager/sql_context.rs b/crates/nu-command/src/dataframe/eager/sql_context.rs index 9e482fcb86..4ae2b77849 100644 --- a/crates/nu-command/src/dataframe/eager/sql_context.rs +++ b/crates/nu-command/src/dataframe/eager/sql_context.rs @@ -1,5 +1,5 @@ use crate::dataframe::eager::sql_expr::parse_sql_expr; -use polars::error::PolarsError; +use polars::error::{ErrString, PolarsError}; use polars::prelude::{col, DataFrame, DataType, IntoLazy, LazyFrame}; use sqlparser::ast::{ Expr as SqlExpr, Select, SelectItem, SetExpr, Statement, TableFactor, Value as SQLValue, @@ -30,7 +30,7 @@ impl SQLContext { // Determine involved dataframe // Implicit join require some more work in query parsers, Explicit join are preferred for now. let tbl = select_stmt.from.get(0).ok_or_else(|| { - PolarsError::NotFound("No table found in select statement".to_string()) + PolarsError::NotFound(ErrString::from("No table found in select statement")) })?; let mut alias_map = HashMap::new(); let tbl_name = match &tbl.relation { @@ -39,7 +39,7 @@ impl SQLContext { .0 .get(0) .ok_or_else(|| { - PolarsError::NotFound("No table found in select statement".to_string()) + PolarsError::NotFound(ErrString::from("No table found in select statement")) })? .value .to_string(); @@ -181,7 +181,7 @@ impl SQLContext { } else { let ast = ast .get(0) - .ok_or_else(|| PolarsError::NotFound("No statement found".to_string()))?; + .ok_or_else(|| PolarsError::NotFound(ErrString::from("No statement found")))?; Ok(match ast { Statement::Query(query) => { let rs = match &*query.body { diff --git a/crates/nu-command/src/dataframe/eager/sql_expr.rs b/crates/nu-command/src/dataframe/eager/sql_expr.rs index d434acd284..5a7f6aadae 100644 --- a/crates/nu-command/src/dataframe/eager/sql_expr.rs +++ b/crates/nu-command/src/dataframe/eager/sql_expr.rs @@ -1,5 +1,5 @@ use polars::error::PolarsError; -use polars::prelude::{col, lit, DataType, Expr, LiteralValue, Result, TimeUnit}; +use polars::prelude::{col, lit, DataType, Expr, LiteralValue, PolarsResult as Result, TimeUnit}; use sqlparser::ast::{ BinaryOperator as SQLBinaryOperator, DataType as SQLDataType, Expr as SqlExpr, diff --git a/crates/nu-command/src/dataframe/expressions/expressions_macro.rs b/crates/nu-command/src/dataframe/expressions/expressions_macro.rs index 95ff88f37a..dda64321a7 100644 --- a/crates/nu-command/src/dataframe/expressions/expressions_macro.rs +++ b/crates/nu-command/src/dataframe/expressions/expressions_macro.rs @@ -69,6 +69,65 @@ macro_rules! expr_command { } } }; + + ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident, $ddof: expr) => { + #[derive(Clone)] + pub struct $command; + + impl Command for $command { + fn name(&self) -> &str { + $name + } + + fn usage(&self) -> &str { + $desc + } + + fn signature(&self) -> Signature { + Signature::build(self.name()) + .input_type(Type::Custom("expression".into())) + .output_type(Type::Custom("expression".into())) + .category(Category::Custom("expression".into())) + } + + fn examples(&self) -> Vec { + $examples + } + + fn run( + &self, + _engine_state: &EngineState, + _stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let expr = NuExpression::try_from_pipeline(input, call.head)?; + let expr: NuExpression = expr.into_polars().$func($ddof).into(); + + Ok(PipelineData::Value( + NuExpression::into_value(expr, call.head), + None, + )) + } + } + + #[cfg(test)] + mod $test { + use super::super::super::test_dataframe::test_dataframe; + use super::*; + use crate::dataframe::lazy::aggregate::LazyAggregate; + use crate::dataframe::lazy::groupby::ToLazyGroupBy; + + #[test] + fn test_examples() { + test_dataframe(vec![ + Box::new($command {}), + Box::new(LazyAggregate {}), + Box::new(ToLazyGroupBy {}), + ]) + } + } + }; } // ExprList command @@ -419,7 +478,8 @@ expr_command!( ), },], std, - test_std + test_std, + 0 ); // ExprVar command @@ -450,5 +510,6 @@ expr_command!( ), },], var, - test_var + test_var, + 0 ); diff --git a/crates/nu-command/src/dataframe/lazy/aggregate.rs b/crates/nu-command/src/dataframe/lazy/aggregate.rs index 1a23d87115..eea21acc85 100644 --- a/crates/nu-command/src/dataframe/lazy/aggregate.rs +++ b/crates/nu-command/src/dataframe/lazy/aggregate.rs @@ -153,8 +153,8 @@ fn get_col_name(expr: &Expr) -> Option { match expr { Expr::Column(column) => Some(column.to_string()), Expr::Agg(agg) => match agg { - polars::prelude::AggExpr::Min(e) - | polars::prelude::AggExpr::Max(e) + polars::prelude::AggExpr::Min { input: e, .. } + | polars::prelude::AggExpr::Max { input: e, .. } | polars::prelude::AggExpr::Median(e) | polars::prelude::AggExpr::NUnique(e) | polars::prelude::AggExpr::First(e) @@ -164,13 +164,11 @@ fn get_col_name(expr: &Expr) -> Option { | polars::prelude::AggExpr::Count(e) | polars::prelude::AggExpr::Sum(e) | polars::prelude::AggExpr::AggGroups(e) - | polars::prelude::AggExpr::Std(e) - | polars::prelude::AggExpr::Var(e) => get_col_name(e.as_ref()), + | polars::prelude::AggExpr::Std(e, _) + | polars::prelude::AggExpr::Var(e, _) => get_col_name(e.as_ref()), polars::prelude::AggExpr::Quantile { expr, .. } => get_col_name(expr.as_ref()), }, - Expr::Reverse(expr) - | Expr::Shift { input: expr, .. } - | Expr::Filter { input: expr, .. } + Expr::Filter { input: expr, .. } | Expr::Slice { input: expr, .. } | Expr::Cast { expr, .. } | Expr::Sort { expr, .. } @@ -179,11 +177,6 @@ fn get_col_name(expr: &Expr) -> Option { | Expr::Exclude(expr, _) | Expr::Alias(expr, _) | Expr::KeepName(expr) - | Expr::Not(expr) - | Expr::IsNotNull(expr) - | Expr::IsNull(expr) - | Expr::Duplicated(expr) - | Expr::IsUnique(expr) | Expr::Explode(expr) => get_col_name(expr.as_ref()), Expr::Ternary { .. } | Expr::AnonymousFunction { .. } diff --git a/crates/nu-command/src/dataframe/lazy/macro_commands.rs b/crates/nu-command/src/dataframe/lazy/macro_commands.rs index d730ed2b49..97f05b1c91 100644 --- a/crates/nu-command/src/dataframe/lazy/macro_commands.rs +++ b/crates/nu-command/src/dataframe/lazy/macro_commands.rs @@ -58,6 +58,56 @@ macro_rules! lazy_command { } } }; + + ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident, $ddot: expr) => { + #[derive(Clone)] + pub struct $command; + + impl Command for $command { + fn name(&self) -> &str { + $name + } + + fn usage(&self) -> &str { + $desc + } + + fn signature(&self) -> Signature { + Signature::build(self.name()) + .input_type(Type::Custom("dataframe".into())) + .output_type(Type::Custom("dataframe".into())) + .category(Category::Custom("lazyframe".into())) + } + + fn examples(&self) -> Vec { + $examples + } + + fn run( + &self, + _engine_state: &EngineState, + _stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; + let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().$func($ddot)); + + Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) + } + } + + #[cfg(test)] + mod $test { + use super::super::super::test_dataframe::test_dataframe; + use super::*; + + #[test] + fn test_examples() { + test_dataframe(vec![Box::new($command {})]) + } + } + }; } // LazyReverse command @@ -232,7 +282,8 @@ lazy_command!( ), },], std, - test_std + test_std, + 1 ); // LazyVar command @@ -254,5 +305,6 @@ lazy_command!( ), },], var, - test_var + test_var, + 1 ); diff --git a/crates/nu-command/src/dataframe/series/indexes/set_with_idx.rs b/crates/nu-command/src/dataframe/series/indexes/set_with_idx.rs index 18ba0ce411..a9c8f30abb 100644 --- a/crates/nu-command/src/dataframe/series/indexes/set_with_idx.rs +++ b/crates/nu-command/src/dataframe/series/indexes/set_with_idx.rs @@ -116,7 +116,7 @@ fn command( ) })? .into_iter() - .filter_map(|val| val.map(|v| v as usize)); + .flatten(); let df = NuDataFrame::try_from_pipeline(input, call.head)?; let series = df.as_series(call.head)?; diff --git a/crates/nu-command/src/dataframe/values/nu_expression/mod.rs b/crates/nu-command/src/dataframe/values/nu_expression/mod.rs index da66259396..2086adacb2 100644 --- a/crates/nu-command/src/dataframe/values/nu_expression/mod.rs +++ b/crates/nu-command/src/dataframe/values/nu_expression/mod.rs @@ -174,16 +174,6 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value { let cols = vec!["expr".to_string(), "value".to_string()]; match expr { - Expr::Not(expr) => { - let expr = expr_to_value(expr.as_ref(), span); - let cols = vec!["expr".into()]; - - Value::Record { - cols, - vals: vec![expr], - span, - } - } Expr::Alias(expr, alias) => { let expr = expr_to_value(expr.as_ref(), span); let alias = Value::String { @@ -280,8 +270,8 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value { } Expr::Agg(agg_expr) => { let value = match agg_expr { - AggExpr::Min(expr) - | AggExpr::Max(expr) + AggExpr::Min { input: expr, .. } + | AggExpr::Max { input: expr, .. } | AggExpr::Median(expr) | AggExpr::NUnique(expr) | AggExpr::First(expr) @@ -291,8 +281,8 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value { | AggExpr::Count(expr) | AggExpr::Sum(expr) | AggExpr::AggGroups(expr) - | AggExpr::Std(expr) - | AggExpr::Var(expr) => expr_to_value(expr.as_ref(), span), + | AggExpr::Std(expr, _) + | AggExpr::Var(expr, _) => expr_to_value(expr.as_ref(), span), AggExpr::Quantile { expr, quantile, @@ -326,26 +316,6 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value { let vals = vec![expr_type, value]; Value::Record { cols, vals, span } } - Expr::IsNotNull(expr) => { - let expr = expr_to_value(expr.as_ref(), span); - let cols = vec!["expr".into()]; - - Value::Record { - cols, - vals: vec![expr], - span, - } - } - Expr::IsNull(expr) => { - let expr = expr_to_value(expr.as_ref(), span); - let cols = vec!["expr".into()]; - - Value::Record { - cols, - vals: vec![expr], - span, - } - } Expr::Count => { let expr = Value::String { val: "count".into(), @@ -372,36 +342,6 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value { span, } } - Expr::Reverse(expr) => { - let expr = expr_to_value(expr.as_ref(), span); - let cols = vec!["expr".into()]; - - Value::Record { - cols, - vals: vec![expr], - span, - } - } - Expr::Duplicated(expr) => { - let expr = expr_to_value(expr.as_ref(), span); - let cols = vec!["expr".into()]; - - Value::Record { - cols, - vals: vec![expr], - span, - } - } - Expr::IsUnique(expr) => { - let expr = expr_to_value(expr.as_ref(), span); - let cols = vec!["expr".into()]; - - Value::Record { - cols, - vals: vec![expr], - span, - } - } Expr::Explode(expr) => { let expr = expr_to_value(expr.as_ref(), span); let cols = vec!["expr".into()]; @@ -511,21 +451,6 @@ pub fn expr_to_value(expr: &Expr, span: Span) -> Value { span, } } - Expr::Shift { input, periods } => { - let expr = expr_to_value(input.as_ref(), span); - let periods = Value::Int { - val: *periods, - span, - }; - - let cols = vec!["expr".into(), "periods".into()]; - - Value::Record { - cols, - vals: vec![expr, periods], - span, - } - } Expr::Filter { input, by } => { let input = expr_to_value(input.as_ref(), span); let by = expr_to_value(by.as_ref(), span); diff --git a/crates/nu-command/src/dataframe/values/nu_when/mod.rs b/crates/nu-command/src/dataframe/values/nu_when/mod.rs index 883eb25218..3cacc7cf54 100644 --- a/crates/nu-command/src/dataframe/values/nu_when/mod.rs +++ b/crates/nu-command/src/dataframe/values/nu_when/mod.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; #[derive(Clone)] pub enum NuWhen { - WhenThen(WhenThen), + WhenThen(Box), WhenThenThen(WhenThenThen), } @@ -27,7 +27,7 @@ impl<'de> Deserialize<'de> for NuWhen { where D: Deserializer<'de>, { - Ok(NuWhen::WhenThen(when(col("a")).then(col("b")))) + Ok(NuWhen::WhenThen(Box::new(when(col("a")).then(col("b"))))) } } @@ -39,7 +39,7 @@ impl fmt::Debug for NuWhen { impl From for NuWhen { fn from(when_then: WhenThen) -> Self { - NuWhen::WhenThen(when_then) + NuWhen::WhenThen(Box::new(when_then)) } }