From 771270d526d3fce17e1650793c16595fdb7ff67c Mon Sep 17 00:00:00 2001 From: Reilly Wood <26268125+rgwood@users.noreply.github.com> Date: Thu, 5 Jan 2023 11:39:54 -0800 Subject: [PATCH] Add Criterion benchmarks for parser (#7686) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR sets up [Criterion](https://github.com/bheisler/criterion.rs) for benchmarking in the main `nu` crate, and adds some simple parser benchmarks. To run the benchmarks, just do `cargo bench` or `cargo bench -- ` in the repo root: ```bash 〉cargo bench -- parse ... Running benches/parser_benchmark.rs (target/release/deps/parser_benchmark-75d224bac82d5b0b) parse_default_env_file time: [221.17 µs 222.34 µs 223.61 µs] Found 8 outliers among 100 measurements (8.00%) 5 (5.00%) high mild 3 (3.00%) high severe parse_default_config_file time: [1.4935 ms 1.4993 ms 1.5059 ms] Found 11 outliers among 100 measurements (11.00%) 7 (7.00%) high mild 4 (4.00%) high severe ``` Existing benchmarks from `nu-plugin` have been moved into the main `nu` crate to keep all our benchmarks in one place. --- Cargo.lock | 96 +++++++++++++------ Cargo.toml | 12 +++ benches/README.md | 7 ++ .../benches => benches}/encoder_benchmark.rs | 0 benches/parser_benchmark.rs | 34 +++++++ crates/nu-plugin/Cargo.toml | 7 -- crates/nu-plugin/README.md | 6 -- crates/nu-utils/src/locale.rs | 1 - 8 files changed, 119 insertions(+), 44 deletions(-) create mode 100644 benches/README.md rename {crates/nu-plugin/benches => benches}/encoder_benchmark.rs (100%) create mode 100644 benches/parser_benchmark.rs delete mode 100644 crates/nu-plugin/README.md diff --git a/Cargo.lock b/Cargo.lock index 751eb7d57a..5e9e36a799 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,6 +80,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "ansi-str" version = "0.5.0" @@ -542,6 +548,33 @@ dependencies = [ "phf_codegen 0.11.1", ] +[[package]] +name = "ciborium" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" + +[[package]] +name = "ciborium-ll" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clang-sys" version = "1.4.0" @@ -555,13 +588,23 @@ dependencies = [ [[package]] name = "clap" -version = "2.34.0" +version = "3.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "bitflags", - "textwrap 0.11.0", - "unicode-width", + "clap_lex", + "indexmap", + "textwrap 0.16.0", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", ] [[package]] @@ -673,15 +716,16 @@ dependencies = [ [[package]] name = "criterion" -version = "0.3.6" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" dependencies = [ + "anes", "atty", "cast", + "ciborium", "clap", "criterion-plot", - "csv", "itertools", "lazy_static", "num-traits", @@ -690,7 +734,6 @@ dependencies = [ "rayon", "regex", "serde", - "serde_cbor", "serde_derive", "serde_json", "tinytemplate", @@ -699,9 +742,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.4.5" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", "itertools", @@ -2486,6 +2529,7 @@ dependencies = [ "assert_cmd", "atty", "chrono", + "criterion", "crossterm 0.24.0", "ctrlc", "hamcrest2", @@ -2756,7 +2800,6 @@ name = "nu-plugin" version = "0.73.1" dependencies = [ "bincode", - "criterion", "nu-engine", "nu-protocol", "rmp", @@ -3160,6 +3203,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "os_str_bytes" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" + [[package]] name = "output_vt100" version = "0.1.3" @@ -4432,16 +4481,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_cbor" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" -dependencies = [ - "half", - "serde", -] - [[package]] name = "serde_derive" version = "1.0.145" @@ -4959,15 +4998,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - [[package]] name = "textwrap" version = "0.15.1" @@ -4979,6 +5009,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + [[package]] name = "thin-slice" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index e6438fdee5..c8232edb8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,6 +83,7 @@ atty = "0.2" nu-test-support = { path="./crates/nu-test-support", version = "0.73.1" } tempfile = "3.2.0" assert_cmd = "2.0.2" +criterion = "0.4" pretty_assertions = "1.0.0" serial_test = "0.8.0" hamcrest2 = "0.3.0" @@ -140,3 +141,14 @@ path = "src/main.rs" # changing versions in each sub-crate of the workspace is tedious [patch.crates-io] # reedline = { git = "https://github.com/nushell/reedline.git", branch = "main" } + +# Criterion benchmarking setup +# Run all benchmarks with `cargo bench` +# Run individual benchmarks like `cargo bench -- ` e.g. `cargo bench -- parse` +[[bench]] +name = "parser_benchmark" +harness = false + +[[bench]] +name = "encoder_benchmark" +harness = false diff --git a/benches/README.md b/benches/README.md new file mode 100644 index 0000000000..af9ff25b56 --- /dev/null +++ b/benches/README.md @@ -0,0 +1,7 @@ +# Criterion benchmarks + +These are benchmarks using [Criterion](https://github.com/bheisler/criterion.rs), a microbenchmarking tool for Rust. + +Run all benchmarks with `cargo bench` + +Or run individual benchmarks like `cargo bench -- ` e.g. `cargo bench -- parse` \ No newline at end of file diff --git a/crates/nu-plugin/benches/encoder_benchmark.rs b/benches/encoder_benchmark.rs similarity index 100% rename from crates/nu-plugin/benches/encoder_benchmark.rs rename to benches/encoder_benchmark.rs diff --git a/benches/parser_benchmark.rs b/benches/parser_benchmark.rs new file mode 100644 index 0000000000..6e939c4c1e --- /dev/null +++ b/benches/parser_benchmark.rs @@ -0,0 +1,34 @@ +use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use nu_parser::parse; +use nu_protocol::{Span, Value}; +use nu_utils::{get_default_config, get_default_env}; + +fn criterion_benchmark(c: &mut Criterion) { + let mut engine_state = nu_command::create_default_context(); + // parsing config.nu breaks without PWD set + engine_state.add_env_var( + "PWD".into(), + Value::string("/some/dir".to_string(), Span::test_data()), + ); + + let default_env = get_default_env().as_bytes(); + c.bench_function("parse_default_env_file", |b| { + b.iter_batched( + || nu_protocol::engine::StateWorkingSet::new(&engine_state), + |mut working_set| parse(&mut working_set, None, default_env, false, &[]), + BatchSize::SmallInput, + ) + }); + + let default_config = get_default_config().as_bytes(); + c.bench_function("parse_default_config_file", |b| { + b.iter_batched( + || nu_protocol::engine::StateWorkingSet::new(&engine_state), + |mut working_set| parse(&mut working_set, None, default_config, false, &[]), + BatchSize::SmallInput, + ) + }); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/crates/nu-plugin/Cargo.toml b/crates/nu-plugin/Cargo.toml index 5e0879a3b3..2538ea55cc 100644 --- a/crates/nu-plugin/Cargo.toml +++ b/crates/nu-plugin/Cargo.toml @@ -15,10 +15,3 @@ serde = { version = "1.0.143" } serde_json = { version = "1.0"} rmp = "0.8.11" rmp-serde = "1.1.0" - -[dev-dependencies] -criterion = "0.3" - -[[bench]] -name = "encoder_benchmark" -harness = false diff --git a/crates/nu-plugin/README.md b/crates/nu-plugin/README.md deleted file mode 100644 index a7d9663b9d..0000000000 --- a/crates/nu-plugin/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# nu-plugin - -## Benchmark -Here is a simple benchmark for different protocol for encoding/decoding nushell table, with different rows and columns. You can simply run `cargo bench` to run benchmark. - -The relative html report is in `target/criterion/report/index.html`. diff --git a/crates/nu-utils/src/locale.rs b/crates/nu-utils/src/locale.rs index 47241618ea..13e59d5a5f 100644 --- a/crates/nu-utils/src/locale.rs +++ b/crates/nu-utils/src/locale.rs @@ -1,6 +1,5 @@ use num_format::Locale; -#[cfg(debug_assertions)] pub const LOCALE_OVERRIDE_ENV_VAR: &str = "NU_TEST_LOCALE_OVERRIDE"; pub fn get_system_locale() -> Locale {