diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index e8a7f55817..a0295b3b4d 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -19,7 +19,7 @@ jobs: # Prevent sudden announcement of a new advisory from failing ci: continue-on-error: true steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.6 - uses: rustsec/audit-check@v1.4.1 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4815491854..9cb92b0902 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,76 +29,50 @@ jobs: # instead of 14 GB) which is too little for us right now. Revisit when `dfr` commands are # removed and we're only building the `polars` plugin instead platform: [windows-latest, macos-13, ubuntu-20.04] - feature: [default, dataframe] - include: - - feature: default - flags: "" - - feature: dataframe - flags: "--features=dataframe" - exclude: - - platform: windows-latest - feature: dataframe - - platform: macos-13 - feature: dataframe runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.6 - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - with: - rustflags: "" - name: cargo fmt run: cargo fmt --all -- --check # If changing these settings also change toolkit.nu - name: Clippy - run: cargo clippy --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- $CLIPPY_OPTIONS + run: cargo clippy --workspace --exclude nu_plugin_* -- $CLIPPY_OPTIONS # In tests we don't have to deny unwrap - name: Clippy of tests - run: cargo clippy --tests --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings + run: cargo clippy --tests --workspace --exclude nu_plugin_* -- -D warnings - name: Clippy of benchmarks - run: cargo clippy --benches --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings + run: cargo clippy --benches --workspace --exclude nu_plugin_* -- -D warnings tests: strategy: fail-fast: true matrix: platform: [windows-latest, macos-latest, ubuntu-20.04] - feature: [default, dataframe] include: - # linux CI cannot handle clipboard feature - default-flags: "" - - platform: ubuntu-20.04 + # linux CI cannot handle clipboard feature + - platform: ubuntu-20.04 default-flags: "--no-default-features --features=default-no-clipboard" - - feature: default - flags: "" - - feature: dataframe - flags: "--features=dataframe" - exclude: - - platform: windows-latest - feature: dataframe - - platform: macos-latest - feature: dataframe runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.6 - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - with: - rustflags: "" - name: Tests - run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }} ${{ matrix.flags }} - + run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }} - name: Check for clean repo shell: bash run: | @@ -121,12 +95,10 @@ jobs: runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.6 - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - with: - rustflags: "" - name: Install Nushell run: cargo install --path . --locked --no-default-features @@ -174,12 +146,10 @@ jobs: runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.6 - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - with: - rustflags: "" - name: Clippy run: cargo clippy --package nu_plugin_* -- $CLIPPY_OPTIONS diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index ab9f93d97d..d95ddd438a 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -27,7 +27,7 @@ jobs: # if: github.repository == 'nushell/nightly' steps: - name: Checkout - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 if: github.repository == 'nushell/nightly' with: ref: main @@ -84,46 +84,35 @@ jobs: include: - target: aarch64-apple-darwin os: macos-latest - target_rustflags: '' - target: x86_64-apple-darwin os: macos-latest - target_rustflags: '' - target: x86_64-pc-windows-msvc extra: 'bin' os: windows-latest - target_rustflags: '' - target: x86_64-pc-windows-msvc extra: msi os: windows-latest - target_rustflags: '' - target: aarch64-pc-windows-msvc extra: 'bin' os: windows-latest - target_rustflags: '' - target: aarch64-pc-windows-msvc extra: msi os: windows-latest - target_rustflags: '' - target: x86_64-unknown-linux-gnu os: ubuntu-20.04 - target_rustflags: '' - target: x86_64-unknown-linux-musl os: ubuntu-20.04 - target_rustflags: '' - target: aarch64-unknown-linux-gnu os: ubuntu-20.04 - target_rustflags: '' - target: armv7-unknown-linux-gnueabihf os: ubuntu-20.04 - target_rustflags: '' - target: riscv64gc-unknown-linux-gnu os: ubuntu-latest - target_rustflags: '' runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.6 with: ref: main fetch-depth: 0 @@ -134,7 +123,7 @@ jobs: - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135` + # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135` with: rustflags: '' @@ -147,12 +136,10 @@ jobs: id: nu run: nu .github/workflows/release-pkg.nu env: - RELEASE_TYPE: standard OS: ${{ matrix.os }} REF: ${{ github.ref }} TARGET: ${{ matrix.target }} _EXTRA_: ${{ matrix.extra }} - TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }} - name: Create an Issue for Release Failure if: ${{ failure() }} @@ -184,122 +171,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - full: - name: Full - needs: prepare - strategy: - fail-fast: false - matrix: - target: - - aarch64-apple-darwin - - x86_64-apple-darwin - - x86_64-pc-windows-msvc - - aarch64-pc-windows-msvc - - x86_64-unknown-linux-gnu - - x86_64-unknown-linux-musl - - aarch64-unknown-linux-gnu - extra: ['bin'] - include: - - target: aarch64-apple-darwin - os: macos-latest - target_rustflags: '--features=dataframe' - - target: x86_64-apple-darwin - os: macos-latest - target_rustflags: '--features=dataframe' - - target: x86_64-pc-windows-msvc - extra: 'bin' - os: windows-latest - target_rustflags: '--features=dataframe' - - target: x86_64-pc-windows-msvc - extra: msi - os: windows-latest - target_rustflags: '--features=dataframe' - - target: aarch64-pc-windows-msvc - extra: 'bin' - os: windows-latest - target_rustflags: '--features=dataframe' - - target: aarch64-pc-windows-msvc - extra: msi - os: windows-latest - target_rustflags: '--features=dataframe' - - target: x86_64-unknown-linux-gnu - os: ubuntu-20.04 - target_rustflags: '--features=dataframe' - - target: x86_64-unknown-linux-musl - os: ubuntu-20.04 - target_rustflags: '--features=dataframe' - - target: aarch64-unknown-linux-gnu - os: ubuntu-20.04 - target_rustflags: '--features=dataframe' - - runs-on: ${{matrix.os}} - - steps: - - uses: actions/checkout@v4.1.5 - with: - ref: main - fetch-depth: 0 - - - name: Update Rust Toolchain Target - run: | - echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml - - - name: Setup Rust toolchain and cache - uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135` - with: - rustflags: '' - - - name: Setup Nushell - uses: hustcer/setup-nu@v3.10 - with: - version: 0.93.0 - - - name: Release Nu Binary - id: nu - run: nu .github/workflows/release-pkg.nu - env: - RELEASE_TYPE: full - OS: ${{ matrix.os }} - REF: ${{ github.ref }} - TARGET: ${{ matrix.target }} - _EXTRA_: ${{ matrix.extra }} - TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }} - - - name: Create an Issue for Release Failure - if: ${{ failure() }} - uses: JasonEtco/create-an-issue@v2.9.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - update_existing: true - search_existing: open - filename: .github/AUTO_ISSUE_TEMPLATE/nightly-build-fail.md - - - name: Set Outputs of Short SHA - id: vars - run: | - echo "date=$(date -u +'%Y-%m-%d')" >> $GITHUB_OUTPUT - sha_short=$(git rev-parse --short HEAD) - echo "sha_short=${sha_short:0:7}" >> $GITHUB_OUTPUT - - # REF: https://github.com/marketplace/actions/gh-release - # Create a release only in nushell/nightly repo - - name: Publish Archive - uses: softprops/action-gh-release@v2.0.5 - if: ${{ startsWith(github.repository, 'nushell/nightly') }} - with: - draft: false - prerelease: true - name: Nu-nightly-${{ steps.vars.outputs.date }}-${{ steps.vars.outputs.sha_short }} - tag_name: nightly-${{ steps.vars.outputs.sha_short }} - body: | - This is a NIGHTLY build of Nushell. - It is NOT recommended for production use. - files: ${{ steps.nu.outputs.archive }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - cleanup: name: Cleanup # Should only run in nushell/nightly repo @@ -310,7 +181,7 @@ jobs: - name: Waiting for Release run: sleep 1800 - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.6 with: ref: main diff --git a/.github/workflows/release-pkg.nu b/.github/workflows/release-pkg.nu index 046ea2475e..da2779f6d1 100755 --- a/.github/workflows/release-pkg.nu +++ b/.github/workflows/release-pkg.nu @@ -9,7 +9,6 @@ # Instructions for manually creating an MSI for Winget Releases when they fail # Added 2022-11-29 when Windows packaging wouldn't work # Updated again on 2023-02-23 because msis are still failing validation -# Update on 2023-10-18 to use RELEASE_TYPE env var to determine if full or not # To run this manual for windows here are the steps I take # checkout the release you want to publish # 1. git checkout 0.86.0 @@ -17,28 +16,26 @@ # 2. $env:CARGO_TARGET_DIR = "" # 2. hide-env CARGO_TARGET_DIR # 3. $env.TARGET = 'x86_64-pc-windows-msvc' -# 4. $env.TARGET_RUSTFLAGS = '' -# 5. $env.GITHUB_WORKSPACE = 'D:\nushell' -# 6. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt' -# 7. $env.OS = 'windows-latest' -# 8. $env.RELEASE_TYPE = '' # There is full and '' for normal releases +# 4. $env.GITHUB_WORKSPACE = 'D:\nushell' +# 5. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt' +# 6. $env.OS = 'windows-latest' # make sure 7z.exe is in your path https://www.7-zip.org/download.html -# 9. $env.Path = ($env.Path | append 'c:\apps\7-zip') +# 7. $env.Path = ($env.Path | append 'c:\apps\7-zip') # make sure aria2c.exe is in your path https://github.com/aria2/aria2 -# 10. $env.Path = ($env.Path | append 'c:\path\to\aria2c') +# 8. $env.Path = ($env.Path | append 'c:\path\to\aria2c') # make sure you have the wixtools installed https://wixtoolset.org/ -# 11. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools') +# 9. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools') # You need to run the release-pkg twice. The first pass, with _EXTRA_ as 'bin', makes the output # folder and builds everything. The second pass, that generates the msi file, with _EXTRA_ as 'msi' -# 12. $env._EXTRA_ = 'bin' -# 13. source .github\workflows\release-pkg.nu -# 14. cd .. -# 15. $env._EXTRA_ = 'msi' -# 16. source .github\workflows\release-pkg.nu +# 10. $env._EXTRA_ = 'bin' +# 11. source .github\workflows\release-pkg.nu +# 12. cd .. +# 13. $env._EXTRA_ = 'msi' +# 14. source .github\workflows\release-pkg.nu # After msi is generated, you have to update winget-pkgs repo, you'll need to patch the release # by deleting the existing msi and uploading this new msi. Then you'll need to update the hash # on the winget-pkgs PR. To generate the hash, run this command -# 17. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256 +# 15. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256 # Then, just take the output and put it in the winget-pkgs PR for the hash on the msi @@ -48,31 +45,15 @@ let os = $env.OS let target = $env.TARGET # Repo source dir like `/home/runner/work/nushell/nushell` let src = $env.GITHUB_WORKSPACE -let flags = $env.TARGET_RUSTFLAGS let dist = $'($env.GITHUB_WORKSPACE)/output' let version = (open Cargo.toml | get package.version) print $'Debugging info:' -print { version: $version, bin: $bin, os: $os, releaseType: $env.RELEASE_TYPE, target: $target, src: $src, flags: $flags, dist: $dist }; hr-line -b - -# Rename the full release name so that we won't break the existing scripts for standard release downloading, such as: -# curl -s https://api.github.com/repos/chmln/sd/releases/latest | grep browser_download_url | cut -d '"' -f 4 | grep x86_64-unknown-linux-musl -const FULL_RLS_NAMING = { - x86_64-apple-darwin: 'x86_64-darwin-full', - aarch64-apple-darwin: 'aarch64-darwin-full', - x86_64-unknown-linux-gnu: 'x86_64-linux-gnu-full', - x86_64-pc-windows-msvc: 'x86_64-windows-msvc-full', - x86_64-unknown-linux-musl: 'x86_64-linux-musl-full', - aarch64-unknown-linux-gnu: 'aarch64-linux-gnu-full', - aarch64-pc-windows-msvc: 'aarch64-windows-msvc-full', - riscv64gc-unknown-linux-gnu: 'riscv64-linux-gnu-full', - armv7-unknown-linux-gnueabihf: 'armv7-linux-gnueabihf-full', -} +print { version: $version, bin: $bin, os: $os, target: $target, src: $src, dist: $dist }; hr-line -b # $env let USE_UBUNTU = $os starts-with ubuntu -let FULL_NAME = $FULL_RLS_NAMING | get -i $target | default 'unknown-target-full' print $'(char nl)Packaging ($bin) v($version) for ($target) in ($src)...'; hr-line -b if not ('Cargo.lock' | path exists) { cargo generate-lockfile } @@ -91,23 +72,23 @@ if $os in ['macos-latest'] or $USE_UBUNTU { 'aarch64-unknown-linux-gnu' => { sudo apt-get install gcc-aarch64-linux-gnu -y $env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = 'aarch64-linux-gnu-gcc' - cargo-build-nu $flags + cargo-build-nu } 'riscv64gc-unknown-linux-gnu' => { sudo apt-get install gcc-riscv64-linux-gnu -y $env.CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER = 'riscv64-linux-gnu-gcc' - cargo-build-nu $flags + cargo-build-nu } 'armv7-unknown-linux-gnueabihf' => { sudo apt-get install pkg-config gcc-arm-linux-gnueabihf -y $env.CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER = 'arm-linux-gnueabihf-gcc' - cargo-build-nu $flags + cargo-build-nu } _ => { # musl-tools to fix 'Failed to find tool. Is `musl-gcc` installed?' # Actually just for x86_64-unknown-linux-musl target if $USE_UBUNTU { sudo apt install musl-tools -y } - cargo-build-nu $flags + cargo-build-nu } } } @@ -116,7 +97,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU { # Build for Windows without static-link-openssl feature # ---------------------------------------------------------------------------- if $os in ['windows-latest'] { - cargo-build-nu $flags + cargo-build-nu } # ---------------------------------------------------------------------------- @@ -162,7 +143,7 @@ cd $dist; print $'(char nl)Creating release archive...'; hr-line if $os in ['macos-latest'] or $USE_UBUNTU { let files = (ls | get name) - let dest = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' } + let dest = $'($bin)-($version)-($target)' let archive = $'($dist)/($dest).tar.gz' mkdir $dest @@ -177,7 +158,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU { } else if $os == 'windows-latest' { - let releaseStem = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' } + let releaseStem = $'($bin)-($version)-($target)' print $'(char nl)Download less related stuffs...'; hr-line aria2c https://github.com/jftuga/less-Windows/releases/download/less-v608/less.exe -o less.exe @@ -214,19 +195,11 @@ if $os in ['macos-latest'] or $USE_UBUNTU { } } -def 'cargo-build-nu' [ options: string ] { - if ($options | str trim | is-empty) { - if $os == 'windows-latest' { - cargo build --release --all --target $target - } else { - cargo build --release --all --target $target --features=static-link-openssl - } +def 'cargo-build-nu' [] { + if $os == 'windows-latest' { + cargo build --release --all --target $target } else { - if $os == 'windows-latest' { - cargo build --release --all --target $target $options - } else { - cargo build --release --all --target $target --features=static-link-openssl $options - } + cargo build --release --all --target $target --features=static-link-openssl } } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ffe653bd22..7d58501044 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,46 +34,35 @@ jobs: include: - target: aarch64-apple-darwin os: macos-latest - target_rustflags: '' - target: x86_64-apple-darwin os: macos-latest - target_rustflags: '' - target: x86_64-pc-windows-msvc extra: 'bin' os: windows-latest - target_rustflags: '' - target: x86_64-pc-windows-msvc extra: msi os: windows-latest - target_rustflags: '' - target: aarch64-pc-windows-msvc extra: 'bin' os: windows-latest - target_rustflags: '' - target: aarch64-pc-windows-msvc extra: msi os: windows-latest - target_rustflags: '' - target: x86_64-unknown-linux-gnu os: ubuntu-20.04 - target_rustflags: '' - target: x86_64-unknown-linux-musl os: ubuntu-20.04 - target_rustflags: '' - target: aarch64-unknown-linux-gnu os: ubuntu-20.04 - target_rustflags: '' - target: armv7-unknown-linux-gnueabihf os: ubuntu-20.04 - target_rustflags: '' - target: riscv64gc-unknown-linux-gnu os: ubuntu-latest - target_rustflags: '' runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.6 - name: Update Rust Toolchain Target run: | @@ -81,7 +70,7 @@ jobs: - name: Setup Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135` + # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135` with: cache: false rustflags: '' @@ -95,102 +84,10 @@ jobs: id: nu run: nu .github/workflows/release-pkg.nu env: - RELEASE_TYPE: standard OS: ${{ matrix.os }} REF: ${{ github.ref }} TARGET: ${{ matrix.target }} _EXTRA_: ${{ matrix.extra }} - TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }} - - # REF: https://github.com/marketplace/actions/gh-release - - name: Publish Archive - uses: softprops/action-gh-release@v2.0.5 - if: ${{ startsWith(github.ref, 'refs/tags/') }} - with: - draft: true - files: ${{ steps.nu.outputs.archive }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - full: - name: Full - - strategy: - fail-fast: false - matrix: - target: - - aarch64-apple-darwin - - x86_64-apple-darwin - - x86_64-pc-windows-msvc - - aarch64-pc-windows-msvc - - x86_64-unknown-linux-gnu - - x86_64-unknown-linux-musl - - aarch64-unknown-linux-gnu - extra: ['bin'] - include: - - target: aarch64-apple-darwin - os: macos-latest - target_rustflags: '--features=dataframe' - - target: x86_64-apple-darwin - os: macos-latest - target_rustflags: '--features=dataframe' - - target: x86_64-pc-windows-msvc - extra: 'bin' - os: windows-latest - target_rustflags: '--features=dataframe' - - target: x86_64-pc-windows-msvc - extra: msi - os: windows-latest - target_rustflags: '--features=dataframe' - - target: aarch64-pc-windows-msvc - extra: 'bin' - os: windows-latest - target_rustflags: '--features=dataframe' - - target: aarch64-pc-windows-msvc - extra: msi - os: windows-latest - target_rustflags: '--features=dataframe' - - target: x86_64-unknown-linux-gnu - os: ubuntu-20.04 - target_rustflags: '--features=dataframe' - - target: x86_64-unknown-linux-musl - os: ubuntu-20.04 - target_rustflags: '--features=dataframe' - - target: aarch64-unknown-linux-gnu - os: ubuntu-20.04 - target_rustflags: '--features=dataframe' - - runs-on: ${{matrix.os}} - - steps: - - uses: actions/checkout@v4.1.5 - - - name: Update Rust Toolchain Target - run: | - echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml - - - name: Setup Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135` - with: - cache: false - rustflags: '' - - - name: Setup Nushell - uses: hustcer/setup-nu@v3.10 - with: - version: 0.93.0 - - - name: Release Nu Binary - id: nu - run: nu .github/workflows/release-pkg.nu - env: - RELEASE_TYPE: full - OS: ${{ matrix.os }} - REF: ${{ github.ref }} - TARGET: ${{ matrix.target }} - _EXTRA_: ${{ matrix.extra }} - TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }} # REF: https://github.com/marketplace/actions/gh-release - name: Publish Archive diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml index dacfb83928..709f09c2e7 100644 --- a/.github/workflows/typos.yml +++ b/.github/workflows/typos.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Actions Repository - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Check spelling uses: crate-ci/typos@v1.21.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d5cf758b56..6d7c256d4c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,11 +71,6 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref cargo run ``` -- Build and run with dataframe support. - ```nushell - cargo run --features=dataframe - ``` - - Run Clippy on Nushell: ```nushell @@ -93,11 +88,6 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref cargo test --workspace ``` - along with dataframe tests - - ```nushell - cargo test --workspace --features=dataframe - ``` or via the `toolkit.nu` command: ```nushell use toolkit.nu test @@ -240,7 +230,7 @@ You can help us to make the review process a smooth experience: - Choose what simplifies having confidence in the conflict resolution and the review. **Merge commits in your branch are OK** in the squash model. - Feel free to notify your reviewers or affected PR authors if your change might cause larger conflicts with another change. - During the rollup of multiple PRs, we may choose to resolve merge conflicts and CI failures ourselves. (Allow maintainers to push to your branch to enable us to do this quickly.) - + ## License We use the [MIT License](https://github.com/nushell/nushell/blob/main/LICENSE) in all of our Nushell projects. If you are including or referencing a crate that uses the [GPL License](https://www.gnu.org/licenses/gpl-3.0.en.html#license-text) unfortunately we will not be able to accept your PR. diff --git a/Cargo.lock b/Cargo.lock index a93547cdd0..7e27f9490e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -377,7 +377,7 @@ dependencies = [ "bitflags 2.5.0", "cexpr", "clang-sys", - "itertools 0.11.0", + "itertools 0.12.1", "lazy_static", "lazycell", "log", @@ -2364,9 +2364,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libmimalloc-sys" -version = "0.1.37" +version = "0.1.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81eb4061c0582dedea1cbc7aff2240300dd6982e0239d1c99e65c1dbf4a30ba7" +checksum = "0e7bb23d733dfcc8af652a78b7bf232f0e967710d044732185e561e47c0336b6" dependencies = [ "cc", "libc", @@ -2620,9 +2620,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.41" +version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f41a2280ded0da56c8cf898babb86e8f10651a34adcfff190ae9a1159c6908d" +checksum = "e9186d86b79b52f4a77af65604b51225e8db1d6ee7e3f41aec1e40829c71a176" dependencies = [ "libmimalloc-sys", ] @@ -2808,7 +2808,7 @@ dependencies = [ [[package]] name = "nu" -version = "0.93.1" +version = "0.94.1" dependencies = [ "assert_cmd", "crossterm", @@ -2821,7 +2821,6 @@ dependencies = [ "nix", "nu-cli", "nu-cmd-base", - "nu-cmd-dataframe", "nu-cmd-extra", "nu-cmd-lang", "nu-cmd-plugin", @@ -2863,7 +2862,7 @@ dependencies = [ [[package]] name = "nu-cli" -version = "0.93.1" +version = "0.94.1" dependencies = [ "chrono", "crossterm", @@ -2886,7 +2885,6 @@ dependencies = [ "nu-test-support", "nu-utils", "once_cell", - "pathdiff", "percent-encoding", "reedline", "rstest", @@ -2899,7 +2897,7 @@ dependencies = [ [[package]] name = "nu-cmd-base" -version = "0.93.1" +version = "0.94.1" dependencies = [ "indexmap", "miette", @@ -2909,32 +2907,9 @@ dependencies = [ "nu-protocol", ] -[[package]] -name = "nu-cmd-dataframe" -version = "0.93.1" -dependencies = [ - "chrono", - "chrono-tz 0.8.6", - "fancy-regex", - "indexmap", - "nu-cmd-lang", - "nu-engine", - "nu-parser", - "nu-protocol", - "num", - "polars", - "polars-arrow", - "polars-io", - "polars-ops", - "polars-plan", - "polars-utils", - "serde", - "sqlparser 0.45.0", -] - [[package]] name = "nu-cmd-extra" -version = "0.93.1" +version = "0.94.1" dependencies = [ "fancy-regex", "heck 0.5.0", @@ -2959,7 +2934,7 @@ dependencies = [ [[package]] name = "nu-cmd-lang" -version = "0.93.1" +version = "0.94.1" dependencies = [ "itertools 0.12.1", "nu-engine", @@ -2971,7 +2946,7 @@ dependencies = [ [[package]] name = "nu-cmd-plugin" -version = "0.93.1" +version = "0.94.1" dependencies = [ "itertools 0.12.1", "nu-engine", @@ -2982,7 +2957,7 @@ dependencies = [ [[package]] name = "nu-color-config" -version = "0.93.1" +version = "0.94.1" dependencies = [ "nu-ansi-term", "nu-engine", @@ -2994,7 +2969,7 @@ dependencies = [ [[package]] name = "nu-command" -version = "0.93.1" +version = "0.94.1" dependencies = [ "alphanumeric-sort", "base64 0.22.1", @@ -3076,6 +3051,7 @@ dependencies = [ "sha2", "sysinfo", "tabled", + "tempfile", "terminal_size", "titlecase", "toml 0.8.12", @@ -3102,7 +3078,7 @@ dependencies = [ [[package]] name = "nu-engine" -version = "0.93.1" +version = "0.94.1" dependencies = [ "nu-glob", "nu-path", @@ -3112,7 +3088,7 @@ dependencies = [ [[package]] name = "nu-explore" -version = "0.93.1" +version = "0.94.1" dependencies = [ "ansi-str", "anyhow", @@ -3137,14 +3113,14 @@ dependencies = [ [[package]] name = "nu-glob" -version = "0.93.1" +version = "0.94.1" dependencies = [ "doc-comment", ] [[package]] name = "nu-json" -version = "0.93.1" +version = "0.94.1" dependencies = [ "linked-hash-map", "num-traits", @@ -3154,7 +3130,7 @@ dependencies = [ [[package]] name = "nu-lsp" -version = "0.93.1" +version = "0.94.1" dependencies = [ "assert-json-diff", "crossbeam-channel", @@ -3175,7 +3151,7 @@ dependencies = [ [[package]] name = "nu-parser" -version = "0.93.1" +version = "0.94.1" dependencies = [ "bytesize", "chrono", @@ -3191,7 +3167,7 @@ dependencies = [ [[package]] name = "nu-path" -version = "0.93.1" +version = "0.94.1" dependencies = [ "dirs-next", "omnipath", @@ -3200,7 +3176,7 @@ dependencies = [ [[package]] name = "nu-plugin" -version = "0.93.1" +version = "0.94.1" dependencies = [ "log", "nix", @@ -3215,7 +3191,7 @@ dependencies = [ [[package]] name = "nu-plugin-core" -version = "0.93.1" +version = "0.94.1" dependencies = [ "interprocess", "log", @@ -3229,7 +3205,7 @@ dependencies = [ [[package]] name = "nu-plugin-engine" -version = "0.93.1" +version = "0.94.1" dependencies = [ "log", "nu-engine", @@ -3244,7 +3220,7 @@ dependencies = [ [[package]] name = "nu-plugin-protocol" -version = "0.93.1" +version = "0.94.1" dependencies = [ "bincode", "nu-protocol", @@ -3256,7 +3232,7 @@ dependencies = [ [[package]] name = "nu-plugin-test-support" -version = "0.93.1" +version = "0.94.1" dependencies = [ "nu-ansi-term", "nu-cmd-lang", @@ -3274,7 +3250,7 @@ dependencies = [ [[package]] name = "nu-pretty-hex" -version = "0.93.1" +version = "0.94.1" dependencies = [ "heapless", "nu-ansi-term", @@ -3283,7 +3259,7 @@ dependencies = [ [[package]] name = "nu-protocol" -version = "0.93.1" +version = "0.94.1" dependencies = [ "brotli 5.0.0", "byte-unit", @@ -3314,7 +3290,7 @@ dependencies = [ [[package]] name = "nu-std" -version = "0.93.1" +version = "0.94.1" dependencies = [ "log", "miette", @@ -3325,9 +3301,10 @@ dependencies = [ [[package]] name = "nu-system" -version = "0.93.1" +version = "0.94.1" dependencies = [ "chrono", + "itertools 0.12.1", "libc", "libproc", "log", @@ -3342,7 +3319,7 @@ dependencies = [ [[package]] name = "nu-table" -version = "0.93.1" +version = "0.94.1" dependencies = [ "fancy-regex", "nu-ansi-term", @@ -3356,7 +3333,7 @@ dependencies = [ [[package]] name = "nu-term-grid" -version = "0.93.1" +version = "0.94.1" dependencies = [ "nu-utils", "unicode-width", @@ -3364,7 +3341,7 @@ dependencies = [ [[package]] name = "nu-test-support" -version = "0.93.1" +version = "0.94.1" dependencies = [ "nu-glob", "nu-path", @@ -3376,7 +3353,7 @@ dependencies = [ [[package]] name = "nu-utils" -version = "0.93.1" +version = "0.94.1" dependencies = [ "crossterm_winapi", "log", @@ -3402,7 +3379,7 @@ dependencies = [ [[package]] name = "nu_plugin_example" -version = "0.93.1" +version = "0.94.1" dependencies = [ "nu-cmd-lang", "nu-plugin", @@ -3412,7 +3389,7 @@ dependencies = [ [[package]] name = "nu_plugin_formats" -version = "0.93.1" +version = "0.94.1" dependencies = [ "eml-parser", "ical", @@ -3425,7 +3402,7 @@ dependencies = [ [[package]] name = "nu_plugin_gstat" -version = "0.93.1" +version = "0.94.1" dependencies = [ "git2", "nu-plugin", @@ -3434,7 +3411,7 @@ dependencies = [ [[package]] name = "nu_plugin_inc" -version = "0.93.1" +version = "0.94.1" dependencies = [ "nu-plugin", "nu-protocol", @@ -3443,12 +3420,13 @@ dependencies = [ [[package]] name = "nu_plugin_polars" -version = "0.93.1" +version = "0.94.1" dependencies = [ "chrono", "chrono-tz 0.9.0", "fancy-regex", "indexmap", + "mimalloc", "nu-cmd-lang", "nu-command", "nu-engine", @@ -3461,7 +3439,6 @@ dependencies = [ "polars", "polars-arrow", "polars-io", - "polars-lazy", "polars-ops", "polars-plan", "polars-utils", @@ -3474,7 +3451,7 @@ dependencies = [ [[package]] name = "nu_plugin_query" -version = "0.93.1" +version = "0.94.1" dependencies = [ "gjson", "nu-plugin", @@ -3486,7 +3463,7 @@ dependencies = [ [[package]] name = "nu_plugin_stress_internals" -version = "0.93.1" +version = "0.94.1" dependencies = [ "interprocess", "serde", @@ -3612,7 +3589,7 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "nuon" -version = "0.93.1" +version = "0.94.1" dependencies = [ "chrono", "fancy-regex", @@ -4893,7 +4870,8 @@ dependencies = [ [[package]] name = "reedline" version = "0.32.0" -source = "git+https://github.com/nushell/reedline?branch=main#a580ea56d4e5a889468b2969d2a1534379504ab6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf59e4c97b5049ba96b052cdb652368305a2eddcbce9bf1c16f9d003139eeea" dependencies = [ "arboard", "chrono", @@ -5476,9 +5454,9 @@ dependencies = [ [[package]] name = "shadow-rs" -version = "0.27.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7960cbd6ba74691bb15e7ebf97f7136bd02d1115f5695a58c1f31d5645750128" +checksum = "1d75516bdaee8f640543ad1f6e292448c23ce57143f812c3736ab4b0874383df" dependencies = [ "const_format", "is_debug", diff --git a/Cargo.toml b/Cargo.toml index 1022075784..025dbc1eb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ license = "MIT" name = "nu" repository = "https://github.com/nushell/nushell" rust-version = "1.77.2" -version = "0.93.1" +version = "0.94.1" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -32,7 +32,6 @@ members = [ "crates/nu-cmd-extra", "crates/nu-cmd-lang", "crates/nu-cmd-plugin", - "crates/nu-cmd-dataframe", "crates/nu-command", "crates/nu-color-config", "crates/nu-explore", @@ -175,25 +174,22 @@ windows = "0.54" winreg = "0.52" [dependencies] -nu-cli = { path = "./crates/nu-cli", version = "0.93.1" } -nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.93.1" } -nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.93.1" } -nu-cmd-plugin = { path = "./crates/nu-cmd-plugin", version = "0.93.1", optional = true } -nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.93.1", features = [ - "dataframe", -], optional = true } -nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.93.1" } -nu-command = { path = "./crates/nu-command", version = "0.93.1" } -nu-engine = { path = "./crates/nu-engine", version = "0.93.1" } -nu-explore = { path = "./crates/nu-explore", version = "0.93.1" } -nu-lsp = { path = "./crates/nu-lsp/", version = "0.93.1" } -nu-parser = { path = "./crates/nu-parser", version = "0.93.1" } -nu-path = { path = "./crates/nu-path", version = "0.93.1" } -nu-plugin-engine = { path = "./crates/nu-plugin-engine", optional = true, version = "0.93.1" } -nu-protocol = { path = "./crates/nu-protocol", version = "0.93.1" } -nu-std = { path = "./crates/nu-std", version = "0.93.1" } -nu-system = { path = "./crates/nu-system", version = "0.93.1" } -nu-utils = { path = "./crates/nu-utils", version = "0.93.1" } +nu-cli = { path = "./crates/nu-cli", version = "0.94.1" } +nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.94.1" } +nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.94.1" } +nu-cmd-plugin = { path = "./crates/nu-cmd-plugin", version = "0.94.1", optional = true } +nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.94.1" } +nu-command = { path = "./crates/nu-command", version = "0.94.1" } +nu-engine = { path = "./crates/nu-engine", version = "0.94.1" } +nu-explore = { path = "./crates/nu-explore", version = "0.94.1" } +nu-lsp = { path = "./crates/nu-lsp/", version = "0.94.1" } +nu-parser = { path = "./crates/nu-parser", version = "0.94.1" } +nu-path = { path = "./crates/nu-path", version = "0.94.1" } +nu-plugin-engine = { path = "./crates/nu-plugin-engine", optional = true, version = "0.94.1" } +nu-protocol = { path = "./crates/nu-protocol", version = "0.94.1" } +nu-std = { path = "./crates/nu-std", version = "0.94.1" } +nu-system = { path = "./crates/nu-system", version = "0.94.1" } +nu-utils = { path = "./crates/nu-utils", version = "0.94.1" } reedline = { workspace = true, features = ["bashisms", "sqlite"] } @@ -201,7 +197,7 @@ crossterm = { workspace = true } ctrlc = { workspace = true } log = { workspace = true } miette = { workspace = true, features = ["fancy-no-backtrace", "fancy"] } -mimalloc = { version = "0.1.37", default-features = false, optional = true } +mimalloc = { version = "0.1.42", default-features = false, optional = true } serde_json = { workspace = true } simplelog = "0.12" time = "0.3" @@ -222,9 +218,9 @@ nix = { workspace = true, default-features = false, features = [ ] } [dev-dependencies] -nu-test-support = { path = "./crates/nu-test-support", version = "0.93.1" } -nu-plugin-protocol = { path = "./crates/nu-plugin-protocol", version = "0.93.1" } -nu-plugin-core = { path = "./crates/nu-plugin-core", version = "0.93.1" } +nu-test-support = { path = "./crates/nu-test-support", version = "0.94.1" } +nu-plugin-protocol = { path = "./crates/nu-plugin-protocol", version = "0.94.1" } +nu-plugin-core = { path = "./crates/nu-plugin-core", version = "0.94.1" } assert_cmd = "2.0" dirs-next = { workspace = true } tango-bench = "0.5" @@ -272,9 +268,6 @@ system-clipboard = [ which-support = ["nu-command/which-support", "nu-cmd-lang/which-support"] trash-support = ["nu-command/trash-support", "nu-cmd-lang/trash-support"] -# Dataframe feature for nushell -dataframe = ["dep:nu-cmd-dataframe", "nu-cmd-lang/dataframe"] - # SQLite commands for nushell sqlite = ["nu-command/sqlite", "nu-cmd-lang/sqlite"] @@ -306,7 +299,7 @@ bench = false # To use a development version of a dependency please use a global override here # changing versions in each sub-crate of the workspace is tedious [patch.crates-io] -reedline = { git = "https://github.com/nushell/reedline", branch = "main" } +# reedline = { git = "https://github.com/nushell/reedline", branch = "main" } # nu-ansi-term = {git = "https://github.com/nushell/nu-ansi-term.git", branch = "main"} # Run all benchmarks with `cargo bench` @@ -318,4 +311,4 @@ harness = false # cargo bench --bench iai_benchmarks -- --save-summary=pretty-json [[bench]] name = "iai_benchmarks" -harness = false +harness = false \ No newline at end of file diff --git a/crates/nu-cli/Cargo.toml b/crates/nu-cli/Cargo.toml index e631f7ccf6..516afd9f0b 100644 --- a/crates/nu-cli/Cargo.toml +++ b/crates/nu-cli/Cargo.toml @@ -5,27 +5,27 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cli" edition = "2021" license = "MIT" name = "nu-cli" -version = "0.93.1" +version = "0.94.1" [lib] bench = false [dev-dependencies] -nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.1" } -nu-command = { path = "../nu-command", version = "0.93.1" } -nu-test-support = { path = "../nu-test-support", version = "0.93.1" } +nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.94.1" } +nu-command = { path = "../nu-command", version = "0.94.1" } +nu-test-support = { path = "../nu-test-support", version = "0.94.1" } rstest = { workspace = true, default-features = false } tempfile = { workspace = true } [dependencies] -nu-cmd-base = { path = "../nu-cmd-base", version = "0.93.1" } -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-path = { path = "../nu-path", version = "0.93.1" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.93.1", optional = true } -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } -nu-utils = { path = "../nu-utils", version = "0.93.1" } -nu-color-config = { path = "../nu-color-config", version = "0.93.1" } +nu-cmd-base = { path = "../nu-cmd-base", version = "0.94.1" } +nu-engine = { path = "../nu-engine", version = "0.94.1" } +nu-path = { path = "../nu-path", version = "0.94.1" } +nu-parser = { path = "../nu-parser", version = "0.94.1" } +nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.94.1", optional = true } +nu-protocol = { path = "../nu-protocol", version = "0.94.1" } +nu-utils = { path = "../nu-utils", version = "0.94.1" } +nu-color-config = { path = "../nu-color-config", version = "0.94.1" } nu-ansi-term = { workspace = true } reedline = { workspace = true, features = ["bashisms", "sqlite"] } @@ -39,7 +39,6 @@ miette = { workspace = true, features = ["fancy-no-backtrace"] } lscolors = { workspace = true, default-features = false, features = ["nu-ansi-term"] } once_cell = { workspace = true } percent-encoding = { workspace = true } -pathdiff = { workspace = true } sysinfo = { workspace = true } unicode-segmentation = { workspace = true } uuid = { workspace = true, features = ["v4"] } @@ -47,4 +46,4 @@ which = { workspace = true } [features] plugin = ["nu-plugin-engine"] -system-clipboard = ["reedline/system_clipboard"] +system-clipboard = ["reedline/system_clipboard"] \ No newline at end of file diff --git a/crates/nu-cli/src/commands/keybindings.rs b/crates/nu-cli/src/commands/keybindings.rs index 469c0f96cd..347ce983ea 100644 --- a/crates/nu-cli/src/commands/keybindings.rs +++ b/crates/nu-cli/src/commands/keybindings.rs @@ -36,16 +36,6 @@ For more information on input and keybindings, check: call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Keybindings.signature(), - &Keybindings.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-cli/src/menus/help_completions.rs b/crates/nu-cli/src/menus/help_completions.rs index b8bdaad435..c9c1b7bf94 100644 --- a/crates/nu-cli/src/menus/help_completions.rs +++ b/crates/nu-cli/src/menus/help_completions.rs @@ -12,50 +12,49 @@ impl NuHelpCompleter { } fn completion_helper(&self, line: &str, pos: usize) -> Vec { - let full_commands = self.0.get_signatures_with_examples(false); let folded_line = line.to_folded_case(); - //Vec<(Signature, Vec, bool, bool)> { - let mut commands = full_commands - .iter() - .filter(|(sig, _, _, _, _)| { - sig.name.to_folded_case().contains(&folded_line) - || sig.usage.to_folded_case().contains(&folded_line) - || sig - .search_terms - .iter() + let mut commands = self + .0 + .get_decls_sorted(false) + .into_iter() + .filter_map(|(_, decl_id)| { + let decl = self.0.get_decl(decl_id); + (decl.name().to_folded_case().contains(&folded_line) + || decl.usage().to_folded_case().contains(&folded_line) + || decl + .search_terms() + .into_iter() .any(|term| term.to_folded_case().contains(&folded_line)) - || sig.extra_usage.to_folded_case().contains(&folded_line) + || decl.extra_usage().to_folded_case().contains(&folded_line)) + .then_some(decl) }) .collect::>(); - commands.sort_by(|(a, _, _, _, _), (b, _, _, _, _)| { - let a_distance = levenshtein_distance(line, &a.name); - let b_distance = levenshtein_distance(line, &b.name); - a_distance.cmp(&b_distance) - }); + commands.sort_by_cached_key(|decl| levenshtein_distance(line, decl.name())); commands .into_iter() - .map(|(sig, examples, _, _, _)| { + .map(|decl| { let mut long_desc = String::new(); - let usage = &sig.usage; + let usage = decl.usage(); if !usage.is_empty() { long_desc.push_str(usage); long_desc.push_str("\r\n\r\n"); } - let extra_usage = &sig.extra_usage; + let extra_usage = decl.extra_usage(); if !extra_usage.is_empty() { long_desc.push_str(extra_usage); long_desc.push_str("\r\n\r\n"); } + let sig = decl.signature(); let _ = write!(long_desc, "Usage:\r\n > {}\r\n", sig.call_signature()); if !sig.named.is_empty() { - long_desc.push_str(&get_flags_section(Some(&*self.0.clone()), sig, |v| { + long_desc.push_str(&get_flags_section(Some(&*self.0.clone()), &sig, |v| { v.to_parsable_string(", ", &self.0.config) })) } @@ -93,13 +92,14 @@ impl NuHelpCompleter { } } - let extra: Vec = examples + let extra: Vec = decl + .examples() .iter() .map(|example| example.example.replace('\n', "\r\n")) .collect(); Suggestion { - value: sig.name.clone(), + value: decl.name().into(), description: Some(long_desc), style: None, extra: Some(extra), diff --git a/crates/nu-cli/src/repl.rs b/crates/nu-cli/src/repl.rs index 29c2f62734..d9aef32bbb 100644 --- a/crates/nu-cli/src/repl.rs +++ b/crates/nu-cli/src/repl.rs @@ -1110,9 +1110,9 @@ fn run_shell_integration_osc9_9(engine_state: &EngineState, stack: &mut Stack, u let start_time = Instant::now(); // Otherwise, communicate the path as OSC 9;9 from ConEmu (often used for spawning new tabs in the same dir) + // This is helpful in Windows Terminal with Duplicate Tab run_ansi_sequence(&format!( - "\x1b]9;9;{}{}\x1b\\", - if path.starts_with('/') { "" } else { "/" }, + "\x1b]9;9;{}\x1b\\", percent_encoding::utf8_percent_encode(&path, percent_encoding::CONTROLS) )); diff --git a/crates/nu-cli/src/util.rs b/crates/nu-cli/src/util.rs index 7ebea0deb2..e4912e012f 100644 --- a/crates/nu-cli/src/util.rs +++ b/crates/nu-cli/src/util.rs @@ -276,8 +276,8 @@ fn evaluate_source( eval_block::(engine_state, stack, &block, input) }?; - let status = if let PipelineData::ByteStream(stream, ..) = pipeline { - stream.print(false)? + let status = if let PipelineData::ByteStream(..) = pipeline { + pipeline.print(engine_state, stack, false, false)? } else { if let Some(hook) = engine_state.get_config().hooks.display_output.clone() { let pipeline = eval_hook( diff --git a/crates/nu-cli/tests/completions/mod.rs b/crates/nu-cli/tests/completions/mod.rs index cb883b67db..a5b0b13aa8 100644 --- a/crates/nu-cli/tests/completions/mod.rs +++ b/crates/nu-cli/tests/completions/mod.rs @@ -292,6 +292,8 @@ fn partial_completions() { // Create the expected values let expected_paths: Vec = vec![ + file(dir.join("partial_a").join("have_ext.exe")), + file(dir.join("partial_a").join("have_ext.txt")), file(dir.join("partial_a").join("hello")), file(dir.join("partial_a").join("hola")), file(dir.join("partial_b").join("hello_b")), @@ -310,6 +312,8 @@ fn partial_completions() { // Create the expected values let expected_paths: Vec = vec![ file(dir.join("partial_a").join("anotherfile")), + file(dir.join("partial_a").join("have_ext.exe")), + file(dir.join("partial_a").join("have_ext.txt")), file(dir.join("partial_a").join("hello")), file(dir.join("partial_a").join("hola")), file(dir.join("partial_b").join("hello_b")), @@ -360,6 +364,34 @@ fn partial_completions() { // Match the results match_suggestions(expected_paths, suggestions); + + // Test completion for all files under directories whose names begin with "pa" + let file_str = file(dir.join("partial_a").join("have")); + let target_file = format!("rm {file_str}"); + let suggestions = completer.complete(&target_file, target_file.len()); + + // Create the expected values + let expected_paths: Vec = vec![ + file(dir.join("partial_a").join("have_ext.exe")), + file(dir.join("partial_a").join("have_ext.txt")), + ]; + + // Match the results + match_suggestions(expected_paths, suggestions); + + // Test completion for all files under directories whose names begin with "pa" + let file_str = file(dir.join("partial_a").join("have_ext.")); + let file_dir = format!("rm {file_str}"); + let suggestions = completer.complete(&file_dir, file_dir.len()); + + // Create the expected values + let expected_paths: Vec = vec![ + file(dir.join("partial_a").join("have_ext.exe")), + file(dir.join("partial_a").join("have_ext.txt")), + ]; + + // Match the results + match_suggestions(expected_paths, suggestions); } #[test] @@ -394,6 +426,13 @@ fn command_ls_with_filecompletion() { ".hidden_folder/".to_string(), ]; + match_suggestions(expected_paths, suggestions); + + let target_dir = "ls custom_completion."; + let suggestions = completer.complete(target_dir, target_dir.len()); + + let expected_paths: Vec = vec!["custom_completion.nu".to_string()]; + match_suggestions(expected_paths, suggestions) } #[test] @@ -428,6 +467,13 @@ fn command_open_with_filecompletion() { ".hidden_folder/".to_string(), ]; + match_suggestions(expected_paths, suggestions); + + let target_dir = "open custom_completion."; + let suggestions = completer.complete(target_dir, target_dir.len()); + + let expected_paths: Vec = vec!["custom_completion.nu".to_string()]; + match_suggestions(expected_paths, suggestions) } diff --git a/crates/nu-cmd-base/Cargo.toml b/crates/nu-cmd-base/Cargo.toml index b163ff447c..1dcd95ba94 100644 --- a/crates/nu-cmd-base/Cargo.toml +++ b/crates/nu-cmd-base/Cargo.toml @@ -5,17 +5,17 @@ edition = "2021" license = "MIT" name = "nu-cmd-base" repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-base" -version = "0.93.1" +version = "0.94.1" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-path = { path = "../nu-path", version = "0.93.1" } -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } +nu-engine = { path = "../nu-engine", version = "0.94.1" } +nu-parser = { path = "../nu-parser", version = "0.94.1" } +nu-path = { path = "../nu-path", version = "0.94.1" } +nu-protocol = { path = "../nu-protocol", version = "0.94.1" } indexmap = { workspace = true } miette = { workspace = true } -[dev-dependencies] +[dev-dependencies] \ No newline at end of file diff --git a/crates/nu-cmd-base/src/util.rs b/crates/nu-cmd-base/src/util.rs index 9c63dec836..559161329b 100644 --- a/crates/nu-cmd-base/src/util.rs +++ b/crates/nu-cmd-base/src/util.rs @@ -25,8 +25,8 @@ pub fn process_range(range: &Range) -> Result<(isize, isize), MakeRangeError> { Range::IntRange(range) => { let start = range.start().try_into().unwrap_or(0); let end = match range.end() { - Bound::Included(v) => v as isize, - Bound::Excluded(v) => (v - 1) as isize, + Bound::Included(v) => (v + 1) as isize, + Bound::Excluded(v) => v as isize, Bound::Unbounded => isize::MAX, }; Ok((start, end)) diff --git a/crates/nu-cmd-dataframe/Cargo.toml b/crates/nu-cmd-dataframe/Cargo.toml deleted file mode 100644 index a156435a8d..0000000000 --- a/crates/nu-cmd-dataframe/Cargo.toml +++ /dev/null @@ -1,75 +0,0 @@ -[package] -authors = ["The Nushell Project Developers"] -description = "Nushell's dataframe commands based on polars." -edition = "2021" -license = "MIT" -name = "nu-cmd-dataframe" -repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-dataframe" -version = "0.93.1" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[lib] -bench = false - -[dependencies] -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } - -# Potential dependencies for extras -chrono = { workspace = true, features = ["std", "unstable-locales"], default-features = false } -chrono-tz = { workspace = true } -fancy-regex = { workspace = true } -indexmap = { workspace = true } -num = { version = "0.4", optional = true } -serde = { workspace = true, features = ["derive"] } -# keep sqlparser at 0.39.0 until we can update polars -sqlparser = { version = "0.45", optional = true } -polars-io = { version = "0.39", features = ["avro"], optional = true } -polars-arrow = { version = "0.39", optional = true } -polars-ops = { version = "0.39", optional = true } -polars-plan = { version = "0.39", features = ["regex"], optional = true } -polars-utils = { version = "0.39", optional = true } - -[dependencies.polars] -features = [ - "arg_where", - "checked_arithmetic", - "concat_str", - "cross_join", - "csv", - "cum_agg", - "dtype-categorical", - "dtype-datetime", - "dtype-struct", - "dtype-i8", - "dtype-i16", - "dtype-u8", - "dtype-u16", - "dynamic_group_by", - "ipc", - "is_in", - "json", - "lazy", - "object", - "parquet", - "random", - "rolling_window", - "rows", - "serde", - "serde-lazy", - "strings", - "temporal", - "to_dummies", -] -default-features = false -optional = true -version = "0.39" - -[features] -dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "polars-utils", "sqlparser"] -default = [] - -[dev-dependencies] -nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.1" } diff --git a/crates/nu-cmd-dataframe/LICENSE b/crates/nu-cmd-dataframe/LICENSE deleted file mode 100644 index ae174e8595..0000000000 --- a/crates/nu-cmd-dataframe/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2019 - 2023 The Nushell Project Developers - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/crates/nu-cmd-dataframe/src/dataframe/README.md b/crates/nu-cmd-dataframe/src/dataframe/README.md deleted file mode 100644 index 593217ede6..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Dataframe - -This dataframe directory holds all of the definitions of the dataframe data structures and commands. - -There are three sections of commands: - -* [eager](./eager) -* [series](./series) -* [values](./values) - -For more details see the -[Nushell book section on dataframes](https://www.nushell.sh/book/dataframes.html) diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/append.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/append.rs deleted file mode 100644 index c0be67ed6e..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/append.rs +++ /dev/null @@ -1,134 +0,0 @@ -use crate::dataframe::values::{Axis, Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct AppendDF; - -impl Command for AppendDF { - fn name(&self) -> &str { - "dfr append" - } - - fn usage(&self) -> &str { - "Appends a new dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("other", SyntaxShape::Any, "dataframe to be appended") - .switch("col", "appends in col orientation", Some('c')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Appends a dataframe as new columns", - example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df); - $a | dfr append $a"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - Column::new( - "a_x".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b_x".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Appends a dataframe merging at the end of columns", - example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df); - $a | dfr append $a --col"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![ - Value::test_int(1), - Value::test_int(3), - Value::test_int(1), - Value::test_int(3), - ], - ), - Column::new( - "b".to_string(), - vec![ - Value::test_int(2), - Value::test_int(4), - Value::test_int(2), - Value::test_int(4), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let other: Value = call.req(engine_state, stack, 0)?; - - let axis = if call.has_flag(engine_state, stack, "col")? { - Axis::Column - } else { - Axis::Row - }; - let df_other = NuDataFrame::try_from_value(other)?; - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - df.append_df(&df_other, axis, call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(AppendDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs deleted file mode 100644 index be9c33a229..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs +++ /dev/null @@ -1,195 +0,0 @@ -use crate::dataframe::values::{str_to_dtype, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::*; - -#[derive(Clone)] -pub struct CastDF; - -impl Command for CastDF { - fn name(&self) -> &str { - "dfr cast" - } - - fn usage(&self) -> &str { - "Cast a column to a different dtype." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .required( - "dtype", - SyntaxShape::String, - "The dtype to cast the column to", - ) - .optional( - "column", - SyntaxShape::String, - "The column to cast. Required when used with a dataframe.", - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Cast a column in a dataframe to a different dtype", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr cast u8 a | dfr schema", - result: Some(Value::record( - record! { - "a" => Value::string("u8", Span::test_data()), - "b" => Value::string("i64", Span::test_data()), - }, - Span::test_data(), - )), - }, - Example { - description: "Cast a column in a lazy dataframe to a different dtype", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr into-lazy | dfr cast u8 a | dfr schema", - result: Some(Value::record( - record! { - "a" => Value::string("u8", Span::test_data()), - "b" => Value::string("i64", Span::test_data()), - }, - Span::test_data(), - )), - }, - Example { - description: "Cast a column in a expression to a different dtype", - example: r#"[[a b]; [1 2] [1 4]] | dfr into-df | dfr group-by a | dfr agg [ (dfr col b | dfr cast u8 | dfr min | dfr as "b_min") ] | dfr schema"#, - result: None - } - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuLazyFrame::can_downcast(&value) { - let (dtype, column_nm) = df_args(engine_state, stack, call)?; - let df = NuLazyFrame::try_from_value(value)?; - command_lazy(call, column_nm, dtype, df) - } else if NuDataFrame::can_downcast(&value) { - let (dtype, column_nm) = df_args(engine_state, stack, call)?; - let df = NuDataFrame::try_from_value(value)?; - command_eager(call, column_nm, dtype, df) - } else { - let dtype: String = call.req(engine_state, stack, 0)?; - let dtype = str_to_dtype(&dtype, call.head)?; - - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().cast(dtype).into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } -} - -fn df_args( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result<(DataType, String), ShellError> { - let dtype = dtype_arg(engine_state, stack, call)?; - let column_nm: String = - call.opt(engine_state, stack, 1)? - .ok_or(ShellError::MissingParameter { - param_name: "column_name".into(), - span: call.head, - })?; - Ok((dtype, column_nm)) -} - -fn dtype_arg( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let dtype: String = call.req(engine_state, stack, 0)?; - str_to_dtype(&dtype, call.head) -} - -fn command_lazy( - call: &Call, - column_nm: String, - dtype: DataType, - lazy: NuLazyFrame, -) -> Result { - let column = col(&column_nm).cast(dtype); - let lazy = lazy.into_polars().with_columns(&[column]); - let lazy = NuLazyFrame::new(false, lazy); - - Ok(PipelineData::Value( - NuLazyFrame::into_value(lazy, call.head)?, - None, - )) -} - -fn command_eager( - call: &Call, - column_nm: String, - dtype: DataType, - nu_df: NuDataFrame, -) -> Result { - let mut df = nu_df.df; - let column = df - .column(&column_nm) - .map_err(|e| ShellError::GenericError { - error: format!("{e}"), - msg: "".into(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let casted = column.cast(&dtype).map_err(|e| ShellError::GenericError { - error: format!("{e}"), - msg: "".into(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let _ = df - .with_column(casted) - .map_err(|e| ShellError::GenericError { - error: format!("{e}"), - msg: "".into(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let df = NuDataFrame::new(false, df); - Ok(PipelineData::Value(df.into_value(call.head), None)) -} - -#[cfg(test)] -mod test { - - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(CastDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/columns.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/columns.rs deleted file mode 100644 index c9167659b5..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/columns.rs +++ /dev/null @@ -1,73 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ColumnsDF; - -impl Command for ColumnsDF { - fn name(&self) -> &str { - "dfr columns" - } - - fn usage(&self) -> &str { - "Show dataframe columns." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type(Type::Custom("dataframe".into()), Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Dataframe columns", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr columns", - result: Some(Value::list( - vec![Value::test_string("a"), Value::test_string("b")], - Span::test_data(), - )), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let names: Vec = df - .as_ref() - .get_column_names() - .iter() - .map(|v| Value::string(*v, call.head)) - .collect(); - - let names = Value::list(names, call.head); - - Ok(PipelineData::Value(names, None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ColumnsDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs deleted file mode 100644 index 8f9d086947..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs +++ /dev/null @@ -1,115 +0,0 @@ -use crate::dataframe::values::{utils::convert_columns, Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct DropDF; - -impl Command for DropDF { - fn name(&self) -> &str { - "dfr drop" - } - - fn usage(&self) -> &str { - "Creates a new dataframe by dropping the selected columns." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest("rest", SyntaxShape::Any, "column names to be dropped") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "drop column a", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr drop a", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let columns: Vec = call.rest(engine_state, stack, 0)?; - let (col_string, col_span) = convert_columns(columns, call.head)?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let new_df = col_string - .first() - .ok_or_else(|| ShellError::GenericError { - error: "Empty names list".into(), - msg: "No column names were found".into(), - span: Some(col_span), - help: None, - inner: vec![], - }) - .and_then(|col| { - df.as_ref() - .drop(&col.item) - .map_err(|e| ShellError::GenericError { - error: "Error dropping column".into(), - msg: e.to_string(), - span: Some(col.span), - help: None, - inner: vec![], - }) - })?; - - // If there are more columns in the drop selection list, these - // are added from the resulting dataframe - col_string - .iter() - .skip(1) - .try_fold(new_df, |new_df, col| { - new_df - .drop(&col.item) - .map_err(|e| ShellError::GenericError { - error: "Error dropping column".into(), - msg: e.to_string(), - span: Some(col.span), - help: None, - inner: vec![], - }) - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(DropDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs deleted file mode 100644 index b2ae6f7cfc..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs +++ /dev/null @@ -1,119 +0,0 @@ -use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::UniqueKeepStrategy; - -#[derive(Clone)] -pub struct DropDuplicates; - -impl Command for DropDuplicates { - fn name(&self) -> &str { - "dfr drop-duplicates" - } - - fn usage(&self) -> &str { - "Drops duplicate values in dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .optional( - "subset", - SyntaxShape::Table(vec![]), - "subset of columns to drop duplicates", - ) - .switch("maintain", "maintain order", Some('m')) - .switch( - "last", - "keeps last duplicate value (by default keeps first)", - Some('l'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "drop duplicates", - example: "[[a b]; [1 2] [3 4] [1 2]] | dfr into-df | dfr drop-duplicates", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(3), Value::test_int(1)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(4), Value::test_int(2)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let columns: Option> = call.opt(engine_state, stack, 0)?; - let (subset, col_span) = match columns { - Some(cols) => { - let (agg_string, col_span) = convert_columns_string(cols, call.head)?; - (Some(agg_string), col_span) - } - None => (None, call.head), - }; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let subset_slice = subset.as_ref().map(|cols| &cols[..]); - - let keep_strategy = if call.has_flag(engine_state, stack, "last")? { - UniqueKeepStrategy::Last - } else { - UniqueKeepStrategy::First - }; - - df.as_ref() - .unique(subset_slice, keep_strategy, None) - .map_err(|e| ShellError::GenericError { - error: "Error dropping duplicates".into(), - msg: e.to_string(), - span: Some(col_span), - help: None, - inner: vec![], - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(DropDuplicates {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs deleted file mode 100644 index 25a3907426..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs +++ /dev/null @@ -1,137 +0,0 @@ -use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct DropNulls; - -impl Command for DropNulls { - fn name(&self) -> &str { - "dfr drop-nulls" - } - - fn usage(&self) -> &str { - "Drops null values in dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .optional( - "subset", - SyntaxShape::Table(vec![]), - "subset of columns to drop nulls", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "drop null values in dataframe", - example: r#"let df = ([[a b]; [1 2] [3 0] [1 2]] | dfr into-df); - let res = ($df.b / $df.b); - let a = ($df | dfr with-column $res --name res); - $a | dfr drop-nulls"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(1)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(2)], - ), - Column::new( - "res".to_string(), - vec![Value::test_int(1), Value::test_int(1)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "drop null values in dataframe", - example: r#"let s = ([1 2 0 0 3 4] | dfr into-df); - ($s / $s) | dfr drop-nulls"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "div_0_0".to_string(), - vec![ - Value::test_int(1), - Value::test_int(1), - Value::test_int(1), - Value::test_int(1), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let columns: Option> = call.opt(engine_state, stack, 0)?; - - let (subset, col_span) = match columns { - Some(cols) => { - let (agg_string, col_span) = convert_columns_string(cols, call.head)?; - (Some(agg_string), col_span) - } - None => (None, call.head), - }; - - let subset_slice = subset.as_ref().map(|cols| &cols[..]); - - df.as_ref() - .drop_nulls(subset_slice) - .map_err(|e| ShellError::GenericError { - error: "Error dropping nulls".into(), - msg: e.to_string(), - span: Some(col_span), - help: None, - inner: vec![], - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::super::WithColumn; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(DropNulls {}), Box::new(WithColumn {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs deleted file mode 100644 index a572a49551..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs +++ /dev/null @@ -1,104 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct DataTypes; - -impl Command for DataTypes { - fn name(&self) -> &str { - "dfr dtypes" - } - - fn usage(&self) -> &str { - "Show dataframe data types." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Dataframe dtypes", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dtypes", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "column".to_string(), - vec![Value::test_string("a"), Value::test_string("b")], - ), - Column::new( - "dtype".to_string(), - vec![Value::test_string("i64"), Value::test_string("i64")], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut dtypes: Vec = Vec::new(); - let names: Vec = df - .as_ref() - .get_column_names() - .iter() - .map(|v| { - let dtype = df - .as_ref() - .column(v) - .expect("using name from list of names from dataframe") - .dtype(); - - let dtype_str = dtype.to_string(); - - dtypes.push(Value::string(dtype_str, call.head)); - - Value::string(*v, call.head) - }) - .collect(); - - let names_col = Column::new("column".to_string(), names); - let dtypes_col = Column::new("dtype".to_string(), dtypes); - - NuDataFrame::try_from_columns(vec![names_col, dtypes_col], None) - .map(|df| PipelineData::Value(df.into_value(call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(DataTypes {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs deleted file mode 100644 index f47f65a004..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs +++ /dev/null @@ -1,107 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::{prelude::*, series::Series}; - -#[derive(Clone)] -pub struct Dummies; - -impl Command for Dummies { - fn name(&self) -> &str { - "dfr dummies" - } - - fn usage(&self) -> &str { - "Creates a new dataframe with dummy variables." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .switch("drop-first", "Drop first row", Some('d')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create new dataframe with dummy variables from a dataframe", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dummies", - result: Some( - NuDataFrame::try_from_series( - vec![ - Series::new("a_1", &[1_u8, 0]), - Series::new("a_3", &[0_u8, 1]), - Series::new("b_2", &[1_u8, 0]), - Series::new("b_4", &[0_u8, 1]), - ], - Span::test_data(), - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Create new dataframe with dummy variables from a series", - example: "[1 2 2 3 3] | dfr into-df | dfr dummies", - result: Some( - NuDataFrame::try_from_series( - vec![ - Series::new("0_1", &[1_u8, 0, 0, 0, 0]), - Series::new("0_2", &[0_u8, 1, 1, 0, 0]), - Series::new("0_3", &[0_u8, 0, 0, 1, 1]), - ], - Span::test_data(), - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let drop_first: bool = call.has_flag(engine_state, stack, "drop-first")?; - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - df.as_ref() - .to_dummies(None, drop_first) - .map_err(|e| ShellError::GenericError { - error: "Error calculating dummies".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The only allowed column types for dummies are String or Int".into()), - inner: vec![], - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Dummies {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs deleted file mode 100644 index 14c86e8c40..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs +++ /dev/null @@ -1,144 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct FirstDF; - -impl Command for FirstDF { - fn name(&self) -> &str { - "dfr first" - } - - fn usage(&self) -> &str { - "Show only the first number of rows or create a first expression" - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .optional( - "rows", - SyntaxShape::Int, - "starting from the front, the number of rows to return", - ) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Return the first row of a dataframe", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_int(1)]), - Column::new("b".to_string(), vec![Value::test_int(2)]), - ], - None, - ) - .expect("should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Return the first two rows of a dataframe", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first 2", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - ], - None, - ) - .expect("should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates a first expression from a column", - example: "dfr col a | dfr first", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let df = NuDataFrame::try_from_value(value)?; - command(engine_state, stack, call, df) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().first().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let rows: Option = call.opt(engine_state, stack, 0)?; - let rows = rows.unwrap_or(1); - - let res = df.as_ref().head(Some(rows)); - Ok(PipelineData::Value( - NuDataFrame::dataframe_into_value(res, call.head), - None, - )) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example}; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(FirstDF {})]); - test_dataframe_example(&mut engine_state, &FirstDF.examples()[0]); - test_dataframe_example(&mut engine_state, &FirstDF.examples()[1]); - } - - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(FirstDF {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &FirstDF.examples()[2]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/get.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/get.rs deleted file mode 100644 index e8cf337864..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/get.rs +++ /dev/null @@ -1,87 +0,0 @@ -use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct GetDF; - -impl Command for GetDF { - fn name(&self) -> &str { - "dfr get" - } - - fn usage(&self) -> &str { - "Creates dataframe with the selected columns." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest("rest", SyntaxShape::Any, "column names to sort dataframe") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns the selected column", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr get a", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let columns: Vec = call.rest(engine_state, stack, 0)?; - let (col_string, col_span) = convert_columns_string(columns, call.head)?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - df.as_ref() - .select(col_string) - .map_err(|e| ShellError::GenericError { - error: "Error selecting columns".into(), - msg: e.to_string(), - span: Some(col_span), - help: None, - inner: vec![], - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs deleted file mode 100644 index ff2c4f98a2..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs +++ /dev/null @@ -1,118 +0,0 @@ -use crate::dataframe::values::{utils::DEFAULT_ROWS, Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LastDF; - -impl Command for LastDF { - fn name(&self) -> &str { - "dfr last" - } - - fn usage(&self) -> &str { - "Creates new dataframe with tail rows or creates a last expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .optional("rows", SyntaxShape::Int, "Number of rows for tail") - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create new dataframe with last rows", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr last 1", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_int(3)]), - Column::new("b".to_string(), vec![Value::test_int(4)]), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates a last expression from a column", - example: "dfr col a | dfr last", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let df = NuDataFrame::try_from_value(value)?; - command(engine_state, stack, call, df) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().last().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let rows: Option = call.opt(engine_state, stack, 0)?; - let rows = rows.unwrap_or(DEFAULT_ROWS); - - let res = df.as_ref().tail(Some(rows)); - Ok(PipelineData::Value( - NuDataFrame::dataframe_into_value(res, call.head), - None, - )) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example}; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(LastDF {})]); - test_dataframe_example(&mut engine_state, &LastDF.examples()[0]); - } - - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(LastDF {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &LastDF.examples()[1]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/list.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/list.rs deleted file mode 100644 index 1cee694180..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/list.rs +++ /dev/null @@ -1,68 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ListDF; - -impl Command for ListDF { - fn name(&self) -> &str { - "dfr ls" - } - - fn usage(&self) -> &str { - "Lists stored dataframes." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()).category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Creates a new dataframe and shows it in the dataframe list", - example: r#"let test = ([[a b];[1 2] [3 4]] | dfr into-df); - ls"#, - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - let mut vals: Vec<(String, Value)> = vec![]; - - for overlay_frame in engine_state.active_overlays(&[]) { - for var in &overlay_frame.vars { - if let Ok(value) = stack.get_var(*var.1, call.head) { - let name = String::from_utf8_lossy(var.0).to_string(); - vals.push((name, value)); - } - } - } - - let vals = vals - .into_iter() - .filter_map(|(name, value)| { - NuDataFrame::try_from_value(value).ok().map(|df| (name, df)) - }) - .map(|(name, df)| { - Value::record( - record! { - "name" => Value::string(name, call.head), - "columns" => Value::int(df.as_ref().width() as i64, call.head), - "rows" => Value::int(df.as_ref().height() as i64, call.head), - }, - call.head, - ) - }) - .collect::>(); - - let list = Value::list(vals, call.head); - - Ok(list.into_pipeline_data()) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/mod.rs deleted file mode 100644 index db7a5c9312..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/mod.rs +++ /dev/null @@ -1,114 +0,0 @@ -mod append; -mod cast; -mod columns; -mod drop; -mod drop_duplicates; -mod drop_nulls; -mod dtypes; -mod dummies; -mod filter_with; -mod first; -mod get; -mod last; -mod list; -mod melt; -mod open; -mod query_df; -mod rename; -mod sample; -mod schema; -mod shape; -mod slice; -mod sql_context; -mod sql_expr; -mod summary; -mod take; -mod to_arrow; -mod to_avro; -mod to_csv; -mod to_df; -mod to_json_lines; -mod to_nu; -mod to_parquet; -mod with_column; - -use nu_protocol::engine::StateWorkingSet; - -pub use self::open::OpenDataFrame; -pub use append::AppendDF; -pub use cast::CastDF; -pub use columns::ColumnsDF; -pub use drop::DropDF; -pub use drop_duplicates::DropDuplicates; -pub use drop_nulls::DropNulls; -pub use dtypes::DataTypes; -pub use dummies::Dummies; -pub use filter_with::FilterWith; -pub use first::FirstDF; -pub use get::GetDF; -pub use last::LastDF; -pub use list::ListDF; -pub use melt::MeltDF; -pub use query_df::QueryDf; -pub use rename::RenameDF; -pub use sample::SampleDF; -pub use schema::SchemaDF; -pub use shape::ShapeDF; -pub use slice::SliceDF; -pub use sql_context::SQLContext; -pub use summary::Summary; -pub use take::TakeDF; -pub use to_arrow::ToArrow; -pub use to_avro::ToAvro; -pub use to_csv::ToCSV; -pub use to_df::ToDataFrame; -pub use to_json_lines::ToJsonLines; -pub use to_nu::ToNu; -pub use to_parquet::ToParquet; -pub use with_column::WithColumn; - -pub fn add_eager_decls(working_set: &mut StateWorkingSet) { - macro_rules! bind_command { - ( $command:expr ) => { - working_set.add_decl(Box::new($command)); - }; - ( $( $command:expr ),* ) => { - $( working_set.add_decl(Box::new($command)); )* - }; - } - - // Dataframe commands - bind_command!( - AppendDF, - CastDF, - ColumnsDF, - DataTypes, - Summary, - DropDF, - DropDuplicates, - DropNulls, - Dummies, - FilterWith, - FirstDF, - GetDF, - LastDF, - ListDF, - MeltDF, - OpenDataFrame, - QueryDf, - RenameDF, - SampleDF, - SchemaDF, - ShapeDF, - SliceDF, - TakeDF, - ToArrow, - ToAvro, - ToCSV, - ToDataFrame, - ToNu, - ToParquet, - ToJsonLines, - WithColumn - ); -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/open.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/open.rs deleted file mode 100644 index 38d0d0c49f..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/open.rs +++ /dev/null @@ -1,518 +0,0 @@ -use crate::dataframe::values::{NuDataFrame, NuLazyFrame, NuSchema}; -use nu_engine::command_prelude::*; - -use polars::prelude::{ - CsvEncoding, CsvReader, IpcReader, JsonFormat, JsonReader, LazyCsvReader, LazyFileListReader, - LazyFrame, ParallelStrategy, ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader, -}; -use polars_io::{avro::AvroReader, HiveOptions}; -use std::{fs::File, io::BufReader, path::PathBuf}; - -#[derive(Clone)] -pub struct OpenDataFrame; - -impl Command for OpenDataFrame { - fn name(&self) -> &str { - "dfr open" - } - - fn usage(&self) -> &str { - "Opens CSV, JSON, JSON lines, arrow, avro, or parquet file to create dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "file", - SyntaxShape::Filepath, - "file path to load values from", - ) - .switch("lazy", "creates a lazy dataframe", Some('l')) - .named( - "type", - SyntaxShape::String, - "File type: csv, tsv, json, parquet, arrow, avro. If omitted, derive from file extension", - Some('t'), - ) - .named( - "delimiter", - SyntaxShape::String, - "file delimiter character. CSV file", - Some('d'), - ) - .switch( - "no-header", - "Indicates if file doesn't have header. CSV file", - None, - ) - .named( - "infer-schema", - SyntaxShape::Number, - "Number of rows to infer the schema of the file. CSV file", - None, - ) - .named( - "skip-rows", - SyntaxShape::Number, - "Number of rows to skip from file. CSV file", - None, - ) - .named( - "columns", - SyntaxShape::List(Box::new(SyntaxShape::String)), - "Columns to be selected from csv file. CSV and Parquet file", - None, - ) - .named( - "schema", - SyntaxShape::Record(vec![]), - r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#, - Some('s') - ) - .input_output_type(Type::Any, Type::Custom("dataframe".into())) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Takes a file name and creates a dataframe", - example: "dfr open test.csv", - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - command(engine_state, stack, call) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let file: Spanned = call.req(engine_state, stack, 0)?; - - let type_option: Option> = call.get_flag(engine_state, stack, "type")?; - - let type_id = match &type_option { - Some(ref t) => Some((t.item.to_owned(), "Invalid type", t.span)), - None => file.item.extension().map(|e| { - ( - e.to_string_lossy().into_owned(), - "Invalid extension", - file.span, - ) - }), - }; - - match type_id { - Some((e, msg, blamed)) => match e.as_str() { - "csv" | "tsv" => from_csv(engine_state, stack, call), - "parquet" | "parq" => from_parquet(engine_state, stack, call), - "ipc" | "arrow" => from_ipc(engine_state, stack, call), - "json" => from_json(engine_state, stack, call), - "jsonl" => from_jsonl(engine_state, stack, call), - "avro" => from_avro(engine_state, stack, call), - _ => Err(ShellError::FileNotFoundCustom { - msg: format!( - "{msg}. Supported values: csv, tsv, parquet, ipc, arrow, json, jsonl, avro" - ), - span: blamed, - }), - }, - None => Err(ShellError::FileNotFoundCustom { - msg: "File without extension".into(), - span: file.span, - }), - } - .map(|value| PipelineData::Value(value, None)) -} - -fn from_parquet( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - if call.has_flag(engine_state, stack, "lazy")? { - let file: String = call.req(engine_state, stack, 0)?; - let args = ScanArgsParquet { - n_rows: None, - cache: true, - parallel: ParallelStrategy::Auto, - rechunk: false, - row_index: None, - low_memory: false, - cloud_options: None, - use_statistics: false, - hive_options: HiveOptions::default(), - }; - - let df: NuLazyFrame = LazyFrame::scan_parquet(file, args) - .map_err(|e| ShellError::GenericError { - error: "Parquet reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - df.into_value(call.head) - } else { - let file: Spanned = call.req(engine_state, stack, 0)?; - let columns: Option> = call.get_flag(engine_state, stack, "columns")?; - - let r = File::open(&file.item).map_err(|e| ShellError::GenericError { - error: "Error opening file".into(), - msg: e.to_string(), - span: Some(file.span), - help: None, - inner: vec![], - })?; - let reader = ParquetReader::new(r); - - let reader = match columns { - None => reader, - Some(columns) => reader.with_columns(Some(columns)), - }; - - let df: NuDataFrame = reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "Parquet reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(df.into_value(call.head)) - } -} - -fn from_avro( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let file: Spanned = call.req(engine_state, stack, 0)?; - let columns: Option> = call.get_flag(engine_state, stack, "columns")?; - - let r = File::open(&file.item).map_err(|e| ShellError::GenericError { - error: "Error opening file".into(), - msg: e.to_string(), - span: Some(file.span), - help: None, - inner: vec![], - })?; - let reader = AvroReader::new(r); - - let reader = match columns { - None => reader, - Some(columns) => reader.with_columns(Some(columns)), - }; - - let df: NuDataFrame = reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "Avro reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(df.into_value(call.head)) -} - -fn from_ipc( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - if call.has_flag(engine_state, stack, "lazy")? { - let file: String = call.req(engine_state, stack, 0)?; - let args = ScanArgsIpc { - n_rows: None, - cache: true, - rechunk: false, - row_index: None, - memory_map: true, - cloud_options: None, - }; - - let df: NuLazyFrame = LazyFrame::scan_ipc(file, args) - .map_err(|e| ShellError::GenericError { - error: "IPC reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - df.into_value(call.head) - } else { - let file: Spanned = call.req(engine_state, stack, 0)?; - let columns: Option> = call.get_flag(engine_state, stack, "columns")?; - - let r = File::open(&file.item).map_err(|e| ShellError::GenericError { - error: "Error opening file".into(), - msg: e.to_string(), - span: Some(file.span), - help: None, - inner: vec![], - })?; - let reader = IpcReader::new(r); - - let reader = match columns { - None => reader, - Some(columns) => reader.with_columns(Some(columns)), - }; - - let df: NuDataFrame = reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "IPC reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(df.into_value(call.head)) - } -} - -fn from_json( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let file: Spanned = call.req(engine_state, stack, 0)?; - let file = File::open(&file.item).map_err(|e| ShellError::GenericError { - error: "Error opening file".into(), - msg: e.to_string(), - span: Some(file.span), - help: None, - inner: vec![], - })?; - let maybe_schema = call - .get_flag(engine_state, stack, "schema")? - .map(|schema| NuSchema::try_from(&schema)) - .transpose()?; - - let buf_reader = BufReader::new(file); - let reader = JsonReader::new(buf_reader); - - let reader = match maybe_schema { - Some(schema) => reader.with_schema(schema.into()), - None => reader, - }; - - let df: NuDataFrame = reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "Json reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(df.into_value(call.head)) -} - -fn from_jsonl( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let infer_schema: Option = call.get_flag(engine_state, stack, "infer-schema")?; - let maybe_schema = call - .get_flag(engine_state, stack, "schema")? - .map(|schema| NuSchema::try_from(&schema)) - .transpose()?; - let file: Spanned = call.req(engine_state, stack, 0)?; - let file = File::open(&file.item).map_err(|e| ShellError::GenericError { - error: "Error opening file".into(), - msg: e.to_string(), - span: Some(file.span), - help: None, - inner: vec![], - })?; - - let buf_reader = BufReader::new(file); - let reader = JsonReader::new(buf_reader) - .with_json_format(JsonFormat::JsonLines) - .infer_schema_len(infer_schema); - - let reader = match maybe_schema { - Some(schema) => reader.with_schema(schema.into()), - None => reader, - }; - - let df: NuDataFrame = reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "Json lines reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(df.into_value(call.head)) -} - -fn from_csv( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let delimiter: Option> = call.get_flag(engine_state, stack, "delimiter")?; - let no_header: bool = call.has_flag(engine_state, stack, "no-header")?; - let infer_schema: Option = call.get_flag(engine_state, stack, "infer-schema")?; - let skip_rows: Option = call.get_flag(engine_state, stack, "skip-rows")?; - let columns: Option> = call.get_flag(engine_state, stack, "columns")?; - - let maybe_schema = call - .get_flag(engine_state, stack, "schema")? - .map(|schema| NuSchema::try_from(&schema)) - .transpose()?; - - if call.has_flag(engine_state, stack, "lazy")? { - let file: String = call.req(engine_state, stack, 0)?; - let csv_reader = LazyCsvReader::new(file); - - let csv_reader = match delimiter { - None => csv_reader, - Some(d) => { - if d.item.len() != 1 { - return Err(ShellError::GenericError { - error: "Incorrect delimiter".into(), - msg: "Delimiter has to be one character".into(), - span: Some(d.span), - help: None, - inner: vec![], - }); - } else { - let delimiter = match d.item.chars().next() { - Some(d) => d as u8, - None => unreachable!(), - }; - csv_reader.with_separator(delimiter) - } - } - }; - - let csv_reader = csv_reader.has_header(!no_header); - - let csv_reader = match maybe_schema { - Some(schema) => csv_reader.with_schema(Some(schema.into())), - None => csv_reader, - }; - - let csv_reader = match infer_schema { - None => csv_reader, - Some(r) => csv_reader.with_infer_schema_length(Some(r)), - }; - - let csv_reader = match skip_rows { - None => csv_reader, - Some(r) => csv_reader.with_skip_rows(r), - }; - - let df: NuLazyFrame = csv_reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "Parquet reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - df.into_value(call.head) - } else { - let file: Spanned = call.req(engine_state, stack, 0)?; - let csv_reader = CsvReader::from_path(&file.item) - .map_err(|e| ShellError::GenericError { - error: "Error creating CSV reader".into(), - msg: e.to_string(), - span: Some(file.span), - help: None, - inner: vec![], - })? - .with_encoding(CsvEncoding::LossyUtf8); - - let csv_reader = match delimiter { - None => csv_reader, - Some(d) => { - if d.item.len() != 1 { - return Err(ShellError::GenericError { - error: "Incorrect delimiter".into(), - msg: "Delimiter has to be one character".into(), - span: Some(d.span), - help: None, - inner: vec![], - }); - } else { - let delimiter = match d.item.chars().next() { - Some(d) => d as u8, - None => unreachable!(), - }; - csv_reader.with_separator(delimiter) - } - } - }; - - let csv_reader = csv_reader.has_header(!no_header); - - let csv_reader = match maybe_schema { - Some(schema) => csv_reader.with_schema(Some(schema.into())), - None => csv_reader, - }; - - let csv_reader = match infer_schema { - None => csv_reader, - Some(r) => csv_reader.infer_schema(Some(r)), - }; - - let csv_reader = match skip_rows { - None => csv_reader, - Some(r) => csv_reader.with_skip_rows(r), - }; - - let csv_reader = match columns { - None => csv_reader, - Some(columns) => csv_reader.with_columns(Some(columns)), - }; - - let df: NuDataFrame = csv_reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "Parquet reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(df.into_value(call.head)) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/query_df.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/query_df.rs deleted file mode 100644 index 4088e00afa..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/query_df.rs +++ /dev/null @@ -1,104 +0,0 @@ -use crate::dataframe::{ - eager::SQLContext, - values::{Column, NuDataFrame, NuLazyFrame}, -}; -use nu_engine::command_prelude::*; - -// attribution: -// sql_context.rs, and sql_expr.rs were copied from polars-sql. thank you. -// maybe we should just use the crate at some point but it's not published yet. -// https://github.com/pola-rs/polars/tree/master/polars-sql - -#[derive(Clone)] -pub struct QueryDf; - -impl Command for QueryDf { - fn name(&self) -> &str { - "dfr query" - } - - fn usage(&self) -> &str { - "Query dataframe using SQL. Note: The dataframe is always named 'df' in your query's from clause." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("sql", SyntaxShape::String, "sql query") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn search_terms(&self) -> Vec<&str> { - vec!["dataframe", "sql", "search"] - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Query dataframe using SQL", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr query 'select a from df'", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let sql_query: String = call.req(engine_state, stack, 0)?; - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut ctx = SQLContext::new(); - ctx.register("df", &df.df); - let df_sql = ctx - .execute(&sql_query) - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - let lazy = NuLazyFrame::new(false, df_sql); - - let eager = lazy.collect(call.head)?; - let value = Value::custom(Box::new(eager), call.head); - - Ok(PipelineData::Value(value, None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(QueryDf {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs deleted file mode 100644 index 0cb75f34f2..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs +++ /dev/null @@ -1,185 +0,0 @@ -use crate::dataframe::{ - utils::extract_strings, - values::{Column, NuDataFrame, NuLazyFrame}, -}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct RenameDF; - -impl Command for RenameDF { - fn name(&self) -> &str { - "dfr rename" - } - - fn usage(&self) -> &str { - "Rename a dataframe column." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "columns", - SyntaxShape::Any, - "Column(s) to be renamed. A string or list of strings", - ) - .required( - "new names", - SyntaxShape::Any, - "New names for the selected column(s). A string or list of strings", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe or lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Renames a series", - example: "[5 6 7 8] | dfr into-df | dfr rename '0' new_name", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "new_name".to_string(), - vec![ - Value::test_int(5), - Value::test_int(6), - Value::test_int(7), - Value::test_int(8), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Renames a dataframe column", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename a a_new", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a_new".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Renames two dataframe columns", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename [a b] [a_new b_new]", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a_new".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b_new".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuLazyFrame::can_downcast(&value) { - let df = NuLazyFrame::try_from_value(value)?; - command_lazy(engine_state, stack, call, df) - } else { - let df = NuDataFrame::try_from_value(value)?; - command_eager(engine_state, stack, call, df) - } - } -} - -fn command_eager( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - mut df: NuDataFrame, -) -> Result { - let columns: Value = call.req(engine_state, stack, 0)?; - let columns = extract_strings(columns)?; - - let new_names: Value = call.req(engine_state, stack, 1)?; - let new_names = extract_strings(new_names)?; - - for (from, to) in columns.iter().zip(new_names.iter()) { - df.as_mut() - .rename(from, to) - .map_err(|e| ShellError::GenericError { - error: "Error renaming".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - } - - Ok(PipelineData::Value(df.into_value(call.head), None)) -} - -fn command_lazy( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - lazy: NuLazyFrame, -) -> Result { - let columns: Value = call.req(engine_state, stack, 0)?; - let columns = extract_strings(columns)?; - - let new_names: Value = call.req(engine_state, stack, 1)?; - let new_names = extract_strings(new_names)?; - - if columns.len() != new_names.len() { - let value: Value = call.req(engine_state, stack, 1)?; - return Err(ShellError::IncompatibleParametersSingle { - msg: "New name list has different size to column list".into(), - span: value.span(), - }); - } - - let lazy = lazy.into_polars(); - let lazy: NuLazyFrame = lazy.rename(&columns, &new_names).into(); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(RenameDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/sample.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/sample.rs deleted file mode 100644 index 2387cca489..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/sample.rs +++ /dev/null @@ -1,127 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::{prelude::NamedFrom, series::Series}; - -#[derive(Clone)] -pub struct SampleDF; - -impl Command for SampleDF { - fn name(&self) -> &str { - "dfr sample" - } - - fn usage(&self) -> &str { - "Create sample dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .named( - "n-rows", - SyntaxShape::Int, - "number of rows to be taken from dataframe", - Some('n'), - ) - .named( - "fraction", - SyntaxShape::Number, - "fraction of dataframe to be taken", - Some('f'), - ) - .named( - "seed", - SyntaxShape::Number, - "seed for the selection", - Some('s'), - ) - .switch("replace", "sample with replace", Some('e')) - .switch("shuffle", "shuffle sample", Some('u')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Sample rows from dataframe", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr sample --n-rows 1", - result: None, // No expected value because sampling is random - }, - Example { - description: "Shows sample row using fraction and replace", - example: - "[[a b]; [1 2] [3 4] [5 6]] | dfr into-df | dfr sample --fraction 0.5 --replace", - result: None, // No expected value because sampling is random - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let rows: Option> = call.get_flag(engine_state, stack, "n-rows")?; - let fraction: Option> = call.get_flag(engine_state, stack, "fraction")?; - let seed: Option = call - .get_flag::(engine_state, stack, "seed")? - .map(|val| val as u64); - let replace: bool = call.has_flag(engine_state, stack, "replace")?; - let shuffle: bool = call.has_flag(engine_state, stack, "shuffle")?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - match (rows, fraction) { - (Some(rows), None) => df - .as_ref() - .sample_n(&Series::new("s", &[rows.item]), replace, shuffle, seed) - .map_err(|e| ShellError::GenericError { - error: "Error creating sample".into(), - msg: e.to_string(), - span: Some(rows.span), - help: None, - inner: vec![], - }), - (None, Some(frac)) => df - .as_ref() - .sample_frac(&Series::new("frac", &[frac.item]), replace, shuffle, seed) - .map_err(|e| ShellError::GenericError { - error: "Error creating sample".into(), - msg: e.to_string(), - span: Some(frac.span), - help: None, - inner: vec![], - }), - (Some(_), Some(_)) => Err(ShellError::GenericError { - error: "Incompatible flags".into(), - msg: "Only one selection criterion allowed".into(), - span: Some(call.head), - help: None, - inner: vec![], - }), - (None, None) => Err(ShellError::GenericError { - error: "No selection".into(), - msg: "No selection criterion was found".into(), - span: Some(call.head), - help: Some("Perhaps you want to use the flag -n or -f".into()), - inner: vec![], - }), - } - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/schema.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/schema.rs deleted file mode 100644 index cf887482bd..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/schema.rs +++ /dev/null @@ -1,112 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct SchemaDF; - -impl Command for SchemaDF { - fn name(&self) -> &str { - "dfr schema" - } - - fn usage(&self) -> &str { - "Show schema for a dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .switch("datatype-list", "creates a lazy dataframe", Some('l')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Dataframe schema", - example: r#"[[a b]; [1 "foo"] [3 "bar"]] | dfr into-df | dfr schema"#, - result: Some(Value::record( - record! { - "a" => Value::string("i64", Span::test_data()), - "b" => Value::string("str", Span::test_data()), - }, - Span::test_data(), - )), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - if call.has_flag(engine_state, stack, "datatype-list")? { - Ok(PipelineData::Value(datatype_list(Span::unknown()), None)) - } else { - command(engine_state, stack, call, input) - } - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let schema = df.schema(); - let value: Value = schema.into(); - Ok(PipelineData::Value(value, None)) -} - -fn datatype_list(span: Span) -> Value { - let types: Vec = [ - ("null", ""), - ("bool", ""), - ("u8", ""), - ("u16", ""), - ("u32", ""), - ("u64", ""), - ("i8", ""), - ("i16", ""), - ("i32", ""), - ("i64", ""), - ("f32", ""), - ("f64", ""), - ("str", ""), - ("binary", ""), - ("date", ""), - ("datetime", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns. Timezone wildcard is *. Other Timezone examples: UTC, America/Los_Angeles."), - ("duration", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns."), - ("time", ""), - ("object", ""), - ("unknown", ""), - ("list", ""), - ] - .iter() - .map(|(dtype, note)| { - Value::record(record! { - "dtype" => Value::string(*dtype, span), - "note" => Value::string(*note, span), - }, - span) - }) - .collect(); - Value::list(types, span) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(SchemaDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/shape.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/shape.rs deleted file mode 100644 index 6e5e7fa9d3..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/shape.rs +++ /dev/null @@ -1,82 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ShapeDF; - -impl Command for ShapeDF { - fn name(&self) -> &str { - "dfr shape" - } - - fn usage(&self) -> &str { - "Shows column and row size for a dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Shows row and column shape", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr shape", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("rows".to_string(), vec![Value::test_int(2)]), - Column::new("columns".to_string(), vec![Value::test_int(2)]), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let rows = Value::int(df.as_ref().height() as i64, call.head); - - let cols = Value::int(df.as_ref().width() as i64, call.head); - - let rows_col = Column::new("rows".to_string(), vec![rows]); - let cols_col = Column::new("columns".to_string(), vec![cols]); - - NuDataFrame::try_from_columns(vec![rows_col, cols_col], None) - .map(|df| PipelineData::Value(df.into_value(call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ShapeDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/slice.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/slice.rs deleted file mode 100644 index 48906cba2c..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/slice.rs +++ /dev/null @@ -1,84 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct SliceDF; - -impl Command for SliceDF { - fn name(&self) -> &str { - "dfr slice" - } - - fn usage(&self) -> &str { - "Creates new dataframe from a slice of rows." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("offset", SyntaxShape::Int, "start of slice") - .required("size", SyntaxShape::Int, "size of slice") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Create new dataframe from a slice of the rows", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr slice 0 1", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_int(1)]), - Column::new("b".to_string(), vec![Value::test_int(2)]), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let offset: i64 = call.req(engine_state, stack, 0)?; - let size: usize = call.req(engine_state, stack, 1)?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let res = df.as_ref().slice(offset, size); - - Ok(PipelineData::Value( - NuDataFrame::dataframe_into_value(res, call.head), - None, - )) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(SliceDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs deleted file mode 100644 index f558904344..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs +++ /dev/null @@ -1,228 +0,0 @@ -use crate::dataframe::eager::sql_expr::parse_sql_expr; -use polars::error::{ErrString, PolarsError}; -use polars::prelude::{col, DataFrame, DataType, IntoLazy, LazyFrame}; -use sqlparser::ast::{ - Expr as SqlExpr, GroupByExpr, Select, SelectItem, SetExpr, Statement, TableFactor, - Value as SQLValue, -}; -use sqlparser::dialect::GenericDialect; -use sqlparser::parser::Parser; -use std::collections::HashMap; - -#[derive(Default)] -pub struct SQLContext { - table_map: HashMap, - dialect: GenericDialect, -} - -impl SQLContext { - pub fn new() -> Self { - Self { - table_map: HashMap::new(), - dialect: GenericDialect, - } - } - - pub fn register(&mut self, name: &str, df: &DataFrame) { - self.table_map.insert(name.to_owned(), df.clone().lazy()); - } - - fn execute_select(&self, select_stmt: &Select) -> Result { - // Determine involved dataframe - // Implicit join require some more work in query parsers, Explicit join are preferred for now. - let tbl = select_stmt.from.first().ok_or_else(|| { - PolarsError::ComputeError(ErrString::from("No table found in select statement")) - })?; - let mut alias_map = HashMap::new(); - let tbl_name = match &tbl.relation { - TableFactor::Table { name, alias, .. } => { - let tbl_name = name - .0 - .first() - .ok_or_else(|| { - PolarsError::ComputeError(ErrString::from( - "No table found in select statement", - )) - })? - .value - .to_string(); - if self.table_map.contains_key(&tbl_name) { - if let Some(alias) = alias { - alias_map.insert(alias.name.value.clone(), tbl_name.to_owned()); - }; - tbl_name - } else { - return Err(PolarsError::ComputeError( - format!("Table name {tbl_name} was not found").into(), - )); - } - } - // Support bare table, optional with alias for now - _ => return Err(PolarsError::ComputeError("Not implemented".into())), - }; - let df = &self.table_map[&tbl_name]; - let mut raw_projection_before_alias: HashMap = HashMap::new(); - let mut contain_wildcard = false; - // Filter Expression - let df = match select_stmt.selection.as_ref() { - Some(expr) => { - let filter_expression = parse_sql_expr(expr)?; - df.clone().filter(filter_expression) - } - None => df.clone(), - }; - // Column Projections - let projection = select_stmt - .projection - .iter() - .enumerate() - .map(|(i, select_item)| { - Ok(match select_item { - SelectItem::UnnamedExpr(expr) => { - let expr = parse_sql_expr(expr)?; - raw_projection_before_alias.insert(format!("{expr:?}"), i); - expr - } - SelectItem::ExprWithAlias { expr, alias } => { - let expr = parse_sql_expr(expr)?; - raw_projection_before_alias.insert(format!("{expr:?}"), i); - expr.alias(&alias.value) - } - SelectItem::QualifiedWildcard(_, _) | SelectItem::Wildcard(_) => { - contain_wildcard = true; - col("*") - } - }) - }) - .collect::, PolarsError>>()?; - // Check for group by - // After projection since there might be number. - let group_by = match &select_stmt.group_by { - GroupByExpr::All => - Err( - PolarsError::ComputeError("Group-By Error: Only positive number or expression are supported, not all".into()) - )?, - GroupByExpr::Expressions(expressions) => expressions - } - .iter() - .map( - |e|match e { - SqlExpr::Value(SQLValue::Number(idx, _)) => { - let idx = match idx.parse::() { - Ok(0)| Err(_) => Err( - PolarsError::ComputeError( - format!("Group-By Error: Only positive number or expression are supported, got {idx}").into() - )), - Ok(idx) => Ok(idx) - }?; - Ok(projection[idx].clone()) - } - SqlExpr::Value(_) => Err( - PolarsError::ComputeError("Group-By Error: Only positive number or expression are supported".into()) - ), - _ => parse_sql_expr(e) - } - ) - .collect::, PolarsError>>()?; - - let df = if group_by.is_empty() { - df.select(projection) - } else { - // check groupby and projection due to difference between SQL and polars - // Return error on wild card, shouldn't process this - if contain_wildcard { - return Err(PolarsError::ComputeError( - "Group-By Error: Can't process wildcard in group-by".into(), - )); - } - // Default polars group by will have group by columns at the front - // need some container to contain position of group by columns and its position - // at the final agg projection, check the schema for the existence of group by column - // and its projections columns, keeping the original index - let (exclude_expr, groupby_pos): (Vec<_>, Vec<_>) = group_by - .iter() - .map(|expr| raw_projection_before_alias.get(&format!("{expr:?}"))) - .enumerate() - .filter(|(_, proj_p)| proj_p.is_some()) - .map(|(gb_p, proj_p)| (*proj_p.unwrap_or(&0), (*proj_p.unwrap_or(&0), gb_p))) - .unzip(); - let (agg_projection, agg_proj_pos): (Vec<_>, Vec<_>) = projection - .iter() - .enumerate() - .filter(|(i, _)| !exclude_expr.contains(i)) - .enumerate() - .map(|(agg_pj, (proj_p, expr))| (expr.clone(), (proj_p, agg_pj + group_by.len()))) - .unzip(); - let agg_df = df.group_by(group_by).agg(agg_projection); - let mut final_proj_pos = groupby_pos - .into_iter() - .chain(agg_proj_pos) - .collect::>(); - - final_proj_pos.sort_by(|(proj_pa, _), (proj_pb, _)| proj_pa.cmp(proj_pb)); - let final_proj = final_proj_pos - .into_iter() - .map(|(_, shm_p)| { - col(agg_df - .clone() - // FIXME: had to do this mess to get get_index to work, not sure why. need help - .collect() - .unwrap_or_default() - .schema() - .get_at_index(shm_p) - .unwrap_or((&"".into(), &DataType::Null)) - .0) - }) - .collect::>(); - agg_df.select(final_proj) - }; - Ok(df) - } - - pub fn execute(&self, query: &str) -> Result { - let ast = Parser::parse_sql(&self.dialect, query) - .map_err(|e| PolarsError::ComputeError(format!("{e:?}").into()))?; - if ast.len() != 1 { - Err(PolarsError::ComputeError( - "One and only one statement at a time please".into(), - )) - } else { - let ast = ast - .first() - .ok_or_else(|| PolarsError::ComputeError(ErrString::from("No statement found")))?; - Ok(match ast { - Statement::Query(query) => { - let rs = match &*query.body { - SetExpr::Select(select_stmt) => self.execute_select(select_stmt)?, - _ => { - return Err(PolarsError::ComputeError( - "INSERT, UPDATE is not supported for polars".into(), - )) - } - }; - match &query.limit { - Some(SqlExpr::Value(SQLValue::Number(nrow, _))) => { - let nrow = nrow.parse().map_err(|err| { - PolarsError::ComputeError( - format!("Conversion Error: {err:?}").into(), - ) - })?; - rs.limit(nrow) - } - None => rs, - _ => { - return Err(PolarsError::ComputeError( - "Only support number argument to LIMIT clause".into(), - )) - } - } - } - _ => { - return Err(PolarsError::ComputeError( - format!("Statement type {ast:?} is not supported").into(), - )) - } - }) - } - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/sql_expr.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/sql_expr.rs deleted file mode 100644 index 9c0728ea5f..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/sql_expr.rs +++ /dev/null @@ -1,200 +0,0 @@ -use polars::error::PolarsError; -use polars::prelude::{col, lit, DataType, Expr, LiteralValue, PolarsResult as Result, TimeUnit}; - -use sqlparser::ast::{ - ArrayElemTypeDef, BinaryOperator as SQLBinaryOperator, DataType as SQLDataType, - Expr as SqlExpr, Function as SQLFunction, Value as SqlValue, WindowType, -}; - -fn map_sql_polars_datatype(data_type: &SQLDataType) -> Result { - Ok(match data_type { - SQLDataType::Char(_) - | SQLDataType::Varchar(_) - | SQLDataType::Uuid - | SQLDataType::Clob(_) - | SQLDataType::Text - | SQLDataType::String(_) => DataType::String, - SQLDataType::Float(_) => DataType::Float32, - SQLDataType::Real => DataType::Float32, - SQLDataType::Double => DataType::Float64, - SQLDataType::TinyInt(_) => DataType::Int8, - SQLDataType::UnsignedTinyInt(_) => DataType::UInt8, - SQLDataType::SmallInt(_) => DataType::Int16, - SQLDataType::UnsignedSmallInt(_) => DataType::UInt16, - SQLDataType::Int(_) => DataType::Int32, - SQLDataType::UnsignedInt(_) => DataType::UInt32, - SQLDataType::BigInt(_) => DataType::Int64, - SQLDataType::UnsignedBigInt(_) => DataType::UInt64, - - SQLDataType::Boolean => DataType::Boolean, - SQLDataType::Date => DataType::Date, - SQLDataType::Time(_, _) => DataType::Time, - SQLDataType::Timestamp(_, _) => DataType::Datetime(TimeUnit::Microseconds, None), - SQLDataType::Interval => DataType::Duration(TimeUnit::Microseconds), - SQLDataType::Array(array_type_def) => match array_type_def { - ArrayElemTypeDef::AngleBracket(inner_type) - | ArrayElemTypeDef::SquareBracket(inner_type) => { - DataType::List(Box::new(map_sql_polars_datatype(inner_type)?)) - } - _ => { - return Err(PolarsError::ComputeError( - "SQL Datatype Array(None) was not supported in polars-sql yet!".into(), - )) - } - }, - _ => { - return Err(PolarsError::ComputeError( - format!("SQL Datatype {data_type:?} was not supported in polars-sql yet!").into(), - )) - } - }) -} - -fn cast_(expr: Expr, data_type: &SQLDataType) -> Result { - let polars_type = map_sql_polars_datatype(data_type)?; - Ok(expr.cast(polars_type)) -} - -fn binary_op_(left: Expr, right: Expr, op: &SQLBinaryOperator) -> Result { - Ok(match op { - SQLBinaryOperator::Plus => left + right, - SQLBinaryOperator::Minus => left - right, - SQLBinaryOperator::Multiply => left * right, - SQLBinaryOperator::Divide => left / right, - SQLBinaryOperator::Modulo => left % right, - SQLBinaryOperator::StringConcat => { - left.cast(DataType::String) + right.cast(DataType::String) - } - SQLBinaryOperator::Gt => left.gt(right), - SQLBinaryOperator::Lt => left.lt(right), - SQLBinaryOperator::GtEq => left.gt_eq(right), - SQLBinaryOperator::LtEq => left.lt_eq(right), - SQLBinaryOperator::Eq => left.eq(right), - SQLBinaryOperator::NotEq => left.eq(right).not(), - SQLBinaryOperator::And => left.and(right), - SQLBinaryOperator::Or => left.or(right), - SQLBinaryOperator::Xor => left.xor(right), - _ => { - return Err(PolarsError::ComputeError( - format!("SQL Operator {op:?} was not supported in polars-sql yet!").into(), - )) - } - }) -} - -fn literal_expr(value: &SqlValue) -> Result { - Ok(match value { - SqlValue::Number(s, _) => { - // Check for existence of decimal separator dot - if s.contains('.') { - s.parse::().map(lit).map_err(|_| { - PolarsError::ComputeError(format!("Can't parse literal {s:?}").into()) - }) - } else { - s.parse::().map(lit).map_err(|_| { - PolarsError::ComputeError(format!("Can't parse literal {s:?}").into()) - }) - }? - } - SqlValue::SingleQuotedString(s) => lit(s.clone()), - SqlValue::NationalStringLiteral(s) => lit(s.clone()), - SqlValue::HexStringLiteral(s) => lit(s.clone()), - SqlValue::DoubleQuotedString(s) => lit(s.clone()), - SqlValue::Boolean(b) => lit(*b), - SqlValue::Null => Expr::Literal(LiteralValue::Null), - _ => { - return Err(PolarsError::ComputeError( - format!("Parsing SQL Value {value:?} was not supported in polars-sql yet!").into(), - )) - } - }) -} - -pub fn parse_sql_expr(expr: &SqlExpr) -> Result { - Ok(match expr { - SqlExpr::Identifier(e) => col(&e.value), - SqlExpr::BinaryOp { left, op, right } => { - let left = parse_sql_expr(left)?; - let right = parse_sql_expr(right)?; - binary_op_(left, right, op)? - } - SqlExpr::Function(sql_function) => parse_sql_function(sql_function)?, - SqlExpr::Cast { - expr, - data_type, - format: _, - } => cast_(parse_sql_expr(expr)?, data_type)?, - SqlExpr::Nested(expr) => parse_sql_expr(expr)?, - SqlExpr::Value(value) => literal_expr(value)?, - _ => { - return Err(PolarsError::ComputeError( - format!("Expression: {expr:?} was not supported in polars-sql yet!").into(), - )) - } - }) -} - -fn apply_window_spec(expr: Expr, window_type: Option<&WindowType>) -> Result { - Ok(match &window_type { - Some(wtype) => match wtype { - WindowType::WindowSpec(window_spec) => { - // Process for simple window specification, partition by first - let partition_by = window_spec - .partition_by - .iter() - .map(parse_sql_expr) - .collect::>>()?; - expr.over(partition_by) - // Order by and Row range may not be supported at the moment - } - // TODO: make NamedWindow work - WindowType::NamedWindow(_named) => { - return Err(PolarsError::ComputeError( - format!("Expression: {expr:?} was not supported in polars-sql yet!").into(), - )) - } - }, - None => expr, - }) -} - -fn parse_sql_function(sql_function: &SQLFunction) -> Result { - use sqlparser::ast::{FunctionArg, FunctionArgExpr}; - // Function name mostly do not have name space, so it mostly take the first args - let function_name = sql_function.name.0[0].value.to_ascii_lowercase(); - let args = sql_function - .args - .iter() - .map(|arg| match arg { - FunctionArg::Named { arg, .. } => arg, - FunctionArg::Unnamed(arg) => arg, - }) - .collect::>(); - Ok( - match ( - function_name.as_str(), - args.as_slice(), - sql_function.distinct, - ) { - ("sum", [FunctionArgExpr::Expr(expr)], false) => { - apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.sum() - } - ("count", [FunctionArgExpr::Expr(expr)], false) => { - apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.count() - } - ("count", [FunctionArgExpr::Expr(expr)], true) => { - apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.n_unique() - } - // Special case for wildcard args to count function. - ("count", [FunctionArgExpr::Wildcard], false) => lit(1i32).count(), - _ => { - return Err(PolarsError::ComputeError( - format!( - "Function {function_name:?} with args {args:?} was not supported in polars-sql yet!" - ) - .into(), - )) - } - }, - ) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/summary.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/summary.rs deleted file mode 100644 index 845929a52d..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/summary.rs +++ /dev/null @@ -1,279 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::{ - chunked_array::ChunkedArray, - prelude::{ - AnyValue, DataFrame, DataType, Float64Type, IntoSeries, NewChunkedArray, - QuantileInterpolOptions, Series, StringType, - }, -}; - -#[derive(Clone)] -pub struct Summary; - -impl Command for Summary { - fn name(&self) -> &str { - "dfr summary" - } - - fn usage(&self) -> &str { - "For a dataframe, produces descriptive statistics (summary statistics) for its numeric columns." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .category(Category::Custom("dataframe".into())) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .named( - "quantiles", - SyntaxShape::Table(vec![]), - "provide optional quantiles", - Some('q'), - ) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "list dataframe descriptives", - example: "[[a b]; [1 1] [1 1]] | dfr into-df | dfr summary", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "descriptor".to_string(), - vec![ - Value::test_string("count"), - Value::test_string("sum"), - Value::test_string("mean"), - Value::test_string("median"), - Value::test_string("std"), - Value::test_string("min"), - Value::test_string("25%"), - Value::test_string("50%"), - Value::test_string("75%"), - Value::test_string("max"), - ], - ), - Column::new( - "a (i64)".to_string(), - vec![ - Value::test_float(2.0), - Value::test_float(2.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(0.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(1.0), - ], - ), - Column::new( - "b (i64)".to_string(), - vec![ - Value::test_float(2.0), - Value::test_float(2.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(0.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(1.0), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let quantiles: Option> = call.get_flag(engine_state, stack, "quantiles")?; - let quantiles = quantiles.map(|values| { - values - .iter() - .map(|value| { - let span = value.span(); - match value { - Value::Float { val, .. } => { - if (&0.0..=&1.0).contains(&val) { - Ok(*val) - } else { - Err(ShellError::GenericError { - error: "Incorrect value for quantile".into(), - msg: "value should be between 0 and 1".into(), - span: Some(span), - help: None, - inner: vec![], - }) - } - } - Value::Error { error, .. } => Err(*error.clone()), - _ => Err(ShellError::GenericError { - error: "Incorrect value for quantile".into(), - msg: "value should be a float".into(), - span: Some(span), - help: None, - inner: vec![], - }), - } - }) - .collect::, ShellError>>() - }); - - let quantiles = match quantiles { - Some(quantiles) => quantiles?, - None => vec![0.25, 0.50, 0.75], - }; - - let mut quantiles_labels = quantiles - .iter() - .map(|q| Some(format!("{}%", q * 100.0))) - .collect::>>(); - let mut labels = vec![ - Some("count".to_string()), - Some("sum".to_string()), - Some("mean".to_string()), - Some("median".to_string()), - Some("std".to_string()), - Some("min".to_string()), - ]; - labels.append(&mut quantiles_labels); - labels.push(Some("max".to_string())); - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let names = ChunkedArray::::from_slice_options("descriptor", &labels).into_series(); - - let head = std::iter::once(names); - - let tail = df - .as_ref() - .get_columns() - .iter() - .filter(|col| !matches!(col.dtype(), &DataType::Object("object", _))) - .map(|col| { - let count = col.len() as f64; - - let sum = col.sum_as_series().ok().and_then(|series| { - series - .cast(&DataType::Float64) - .ok() - .and_then(|ca| match ca.get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }) - }); - - let mean = match col.mean_as_series().get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }; - - let median = match col.median_as_series() { - Ok(v) => match v.get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }, - _ => None, - }; - - let std = match col.std_as_series(0) { - Ok(v) => match v.get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }, - _ => None, - }; - - let min = col.min_as_series().ok().and_then(|series| { - series - .cast(&DataType::Float64) - .ok() - .and_then(|ca| match ca.get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }) - }); - - let mut quantiles = quantiles - .clone() - .into_iter() - .map(|q| { - col.quantile_as_series(q, QuantileInterpolOptions::default()) - .ok() - .and_then(|ca| ca.cast(&DataType::Float64).ok()) - .and_then(|ca| match ca.get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }) - }) - .collect::>>(); - - let max = col.max_as_series().ok().and_then(|series| { - series - .cast(&DataType::Float64) - .ok() - .and_then(|ca| match ca.get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }) - }); - - let mut descriptors = vec![Some(count), sum, mean, median, std, min]; - descriptors.append(&mut quantiles); - descriptors.push(max); - - let name = format!("{} ({})", col.name(), col.dtype()); - ChunkedArray::::from_slice_options(&name, &descriptors).into_series() - }); - - let res = head.chain(tail).collect::>(); - - DataFrame::new(res) - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Summary {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/take.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/take.rs deleted file mode 100644 index 406dd1d624..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/take.rs +++ /dev/null @@ -1,148 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::DataType; - -#[derive(Clone)] -pub struct TakeDF; - -impl Command for TakeDF { - fn name(&self) -> &str { - "dfr take" - } - - fn usage(&self) -> &str { - "Creates new dataframe using the given indices." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "indices", - SyntaxShape::Any, - "list of indices used to take data", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Takes selected rows from dataframe", - example: r#"let df = ([[a b]; [4 1] [5 2] [4 3]] | dfr into-df); - let indices = ([0 2] | dfr into-df); - $df | dfr take $indices"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(4), Value::test_int(4)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Takes selected rows from series", - example: r#"let series = ([4 1 5 2 4 3] | dfr into-df); - let indices = ([0 2] | dfr into-df); - $series | dfr take $indices"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(4), Value::test_int(5)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let index_value: Value = call.req(engine_state, stack, 0)?; - let index_span = index_value.span(); - let index = NuDataFrame::try_from_value(index_value)?.as_series(index_span)?; - - let casted = match index.dtype() { - DataType::UInt32 | DataType::UInt64 | DataType::Int32 | DataType::Int64 => index - .cast(&DataType::UInt32) - .map_err(|e| ShellError::GenericError { - error: "Error casting index list".into(), - msg: e.to_string(), - span: Some(index_span), - help: None, - inner: vec![], - }), - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: "Series with incorrect type".into(), - span: Some(call.head), - help: Some("Consider using a Series with type int type".into()), - inner: vec![], - }), - }?; - - let indices = casted.u32().map_err(|e| ShellError::GenericError { - error: "Error casting index list".into(), - msg: e.to_string(), - span: Some(index_span), - help: None, - inner: vec![], - })?; - - NuDataFrame::try_from_pipeline(input, call.head).and_then(|df| { - df.as_ref() - .take(indices) - .map_err(|e| ShellError::GenericError { - error: "Error taking values".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) - }) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(TakeDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_arrow.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_arrow.rs deleted file mode 100644 index 66f13121bf..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_arrow.rs +++ /dev/null @@ -1,79 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::prelude::{IpcWriter, SerWriter}; -use std::{fs::File, path::PathBuf}; - -#[derive(Clone)] -pub struct ToArrow; - -impl Command for ToArrow { - fn name(&self) -> &str { - "dfr to-arrow" - } - - fn usage(&self) -> &str { - "Saves dataframe to arrow file." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("file", SyntaxShape::Filepath, "file path to save dataframe") - .input_output_type(Type::Custom("dataframe".into()), Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Saves dataframe to arrow file", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-arrow test.arrow", - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let file_name: Spanned = call.req(engine_state, stack, 0)?; - - let mut df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut file = File::create(&file_name.item).map_err(|e| ShellError::GenericError { - error: "Error with file name".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - IpcWriter::new(&mut file) - .finish(df.as_mut()) - .map_err(|e| ShellError::GenericError { - error: "Error saving file".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span); - - Ok(PipelineData::Value( - Value::list(vec![file_value], call.head), - None, - )) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_avro.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_avro.rs deleted file mode 100644 index e5e5c6fae1..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_avro.rs +++ /dev/null @@ -1,109 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars_io::{ - avro::{AvroCompression, AvroWriter}, - SerWriter, -}; -use std::{fs::File, path::PathBuf}; - -#[derive(Clone)] -pub struct ToAvro; - -impl Command for ToAvro { - fn name(&self) -> &str { - "dfr to-avro" - } - - fn usage(&self) -> &str { - "Saves dataframe to avro file." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .named( - "compression", - SyntaxShape::String, - "use compression, supports deflate or snappy", - Some('c'), - ) - .required("file", SyntaxShape::Filepath, "file path to save dataframe") - .input_output_type(Type::Custom("dataframe".into()), Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Saves dataframe to avro file", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-avro test.avro", - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn get_compression(call: &Call) -> Result, ShellError> { - if let Some((compression, span)) = call - .get_flag_expr("compression") - .and_then(|e| e.as_string().map(|s| (s, e.span))) - { - match compression.as_ref() { - "snappy" => Ok(Some(AvroCompression::Snappy)), - "deflate" => Ok(Some(AvroCompression::Deflate)), - _ => Err(ShellError::IncorrectValue { - msg: "compression must be one of deflate or snappy".to_string(), - val_span: span, - call_span: span, - }), - } - } else { - Ok(None) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let file_name: Spanned = call.req(engine_state, stack, 0)?; - let compression = get_compression(call)?; - - let mut df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError { - error: "Error with file name".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - AvroWriter::new(file) - .with_compression(compression) - .finish(df.as_mut()) - .map_err(|e| ShellError::GenericError { - error: "Error saving file".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span); - - Ok(PipelineData::Value( - Value::list(vec![file_value], call.head), - None, - )) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_csv.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_csv.rs deleted file mode 100644 index d85bed5150..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_csv.rs +++ /dev/null @@ -1,125 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::prelude::{CsvWriter, SerWriter}; -use std::{fs::File, path::PathBuf}; - -#[derive(Clone)] -pub struct ToCSV; - -impl Command for ToCSV { - fn name(&self) -> &str { - "dfr to-csv" - } - - fn usage(&self) -> &str { - "Saves dataframe to CSV file." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("file", SyntaxShape::Filepath, "file path to save dataframe") - .named( - "delimiter", - SyntaxShape::String, - "file delimiter character", - Some('d'), - ) - .switch("no-header", "Indicates if file doesn't have header", None) - .input_output_type(Type::Custom("dataframe".into()), Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Saves dataframe to CSV file", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-csv test.csv", - result: None, - }, - Example { - description: "Saves dataframe to CSV file using other delimiter", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-csv test.csv --delimiter '|'", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let file_name: Spanned = call.req(engine_state, stack, 0)?; - let delimiter: Option> = call.get_flag(engine_state, stack, "delimiter")?; - let no_header: bool = call.has_flag(engine_state, stack, "no-header")?; - - let mut df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut file = File::create(&file_name.item).map_err(|e| ShellError::GenericError { - error: "Error with file name".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - let writer = CsvWriter::new(&mut file); - - let writer = if no_header { - writer.include_header(false) - } else { - writer.include_header(true) - }; - - let mut writer = match delimiter { - None => writer, - Some(d) => { - if d.item.len() != 1 { - return Err(ShellError::GenericError { - error: "Incorrect delimiter".into(), - msg: "Delimiter has to be one char".into(), - span: Some(d.span), - help: None, - inner: vec![], - }); - } else { - let delimiter = match d.item.chars().next() { - Some(d) => d as u8, - None => unreachable!(), - }; - - writer.with_separator(delimiter) - } - } - }; - - writer - .finish(df.as_mut()) - .map_err(|e| ShellError::GenericError { - error: "Error writing to file".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span); - - Ok(PipelineData::Value( - Value::list(vec![file_value], call.head), - None, - )) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_df.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_df.rs deleted file mode 100644 index d768c7a742..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_df.rs +++ /dev/null @@ -1,189 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuSchema}; -use nu_engine::command_prelude::*; - -use polars::prelude::*; - -#[derive(Clone)] -pub struct ToDataFrame; - -impl Command for ToDataFrame { - fn name(&self) -> &str { - "dfr into-df" - } - - fn usage(&self) -> &str { - "Converts a list, table or record into a dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .named( - "schema", - SyntaxShape::Record(vec![]), - r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#, - Some('s'), - ) - .input_output_type(Type::Any, Type::Custom("dataframe".into())) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Takes a dictionary and creates a dataframe", - example: "[[a b];[1 2] [3 4]] | dfr into-df", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Takes a list of tables and creates a dataframe", - example: "[[1 2 a] [3 4 b] [5 6 c]] | dfr into-df", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "0".to_string(), - vec![Value::test_int(1), Value::test_int(3), Value::test_int(5)], - ), - Column::new( - "1".to_string(), - vec![Value::test_int(2), Value::test_int(4), Value::test_int(6)], - ), - Column::new( - "2".to_string(), - vec![ - Value::test_string("a"), - Value::test_string("b"), - Value::test_string("c"), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Takes a list and creates a dataframe", - example: "[a b c] | dfr into-df", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("a"), - Value::test_string("b"), - Value::test_string("c"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Takes a list of booleans and creates a dataframe", - example: "[true true false] | dfr into-df", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Convert to a dataframe and provide a schema", - example: "{a: 1, b: {a: [1 2 3]}, c: [a b c]}| dfr into-df -s {a: u8, b: {a: list}, c: list}", - result: Some( - NuDataFrame::try_from_series(vec![ - Series::new("a", &[1u8]), - { - let dtype = DataType::Struct(vec![Field::new("a", DataType::List(Box::new(DataType::UInt64)))]); - let vals = vec![AnyValue::StructOwned( - Box::new((vec![AnyValue::List(Series::new("a", &[1u64, 2, 3]))], vec![Field::new("a", DataType::String)]))); 1]; - Series::from_any_values_and_dtype("b", &vals, &dtype, false) - .expect("Struct series should not fail") - }, - { - let dtype = DataType::List(Box::new(DataType::String)); - let vals = vec![AnyValue::List(Series::new("c", &["a", "b", "c"]))]; - Series::from_any_values_and_dtype("c", &vals, &dtype, false) - .expect("List series should not fail") - } - ], Span::test_data()) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Convert to a dataframe and provide a schema that adds a new column", - example: r#"[[a b]; [1 "foo"] [2 "bar"]] | dfr into-df -s {a: u8, b:str, c:i64} | dfr fill-null 3"#, - result: Some(NuDataFrame::try_from_series(vec![ - Series::new("a", [1u8, 2]), - Series::new("b", ["foo", "bar"]), - Series::new("c", [3i64, 3]), - ], Span::test_data()) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - } - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let maybe_schema = call - .get_flag(engine_state, stack, "schema")? - .map(|schema| NuSchema::try_from(&schema)) - .transpose()?; - - let df = NuDataFrame::try_from_iter(input.into_iter(), maybe_schema.clone())?; - - Ok(PipelineData::Value( - NuDataFrame::into_value(df, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ToDataFrame {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_json_lines.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_json_lines.rs deleted file mode 100644 index 5875f17107..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_json_lines.rs +++ /dev/null @@ -1,80 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::prelude::{JsonWriter, SerWriter}; -use std::{fs::File, io::BufWriter, path::PathBuf}; - -#[derive(Clone)] -pub struct ToJsonLines; - -impl Command for ToJsonLines { - fn name(&self) -> &str { - "dfr to-jsonl" - } - - fn usage(&self) -> &str { - "Saves dataframe to a JSON lines file." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("file", SyntaxShape::Filepath, "file path to save dataframe") - .input_output_type(Type::Custom("dataframe".into()), Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Saves dataframe to JSON lines file", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-jsonl test.jsonl", - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let file_name: Spanned = call.req(engine_state, stack, 0)?; - - let mut df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError { - error: "Error with file name".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - let buf_writer = BufWriter::new(file); - - JsonWriter::new(buf_writer) - .finish(df.as_mut()) - .map_err(|e| ShellError::GenericError { - error: "Error saving file".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span); - - Ok(PipelineData::Value( - Value::list(vec![file_value], call.head), - None, - )) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs deleted file mode 100644 index a6ab42052c..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs +++ /dev/null @@ -1,136 +0,0 @@ -use crate::dataframe::values::{NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ToNu; - -impl Command for ToNu { - fn name(&self) -> &str { - "dfr into-nu" - } - - fn usage(&self) -> &str { - "Converts a dataframe or an expression into into nushell value for access and exploration." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .named( - "rows", - SyntaxShape::Number, - "number of rows to be shown", - Some('n'), - ) - .switch("tail", "shows tail rows", Some('t')) - .input_output_types(vec![ - (Type::Custom("expression".into()), Type::Any), - (Type::Custom("dataframe".into()), Type::table()), - ]) - //.input_output_type(Type::Any, Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - let rec_1 = Value::test_record(record! { - "index" => Value::test_int(0), - "a" => Value::test_int(1), - "b" => Value::test_int(2), - }); - let rec_2 = Value::test_record(record! { - "index" => Value::test_int(1), - "a" => Value::test_int(3), - "b" => Value::test_int(4), - }); - let rec_3 = Value::test_record(record! { - "index" => Value::test_int(2), - "a" => Value::test_int(3), - "b" => Value::test_int(4), - }); - - vec![ - Example { - description: "Shows head rows from dataframe", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr into-nu", - result: Some(Value::list(vec![rec_1, rec_2], Span::test_data())), - }, - Example { - description: "Shows tail rows from dataframe", - example: "[[a b]; [1 2] [5 6] [3 4]] | dfr into-df | dfr into-nu --tail --rows 1", - result: Some(Value::list(vec![rec_3], Span::test_data())), - }, - Example { - description: "Convert a col expression into a nushell value", - example: "dfr col a | dfr into-nu", - result: Some(Value::test_record(record! { - "expr" => Value::test_string("column"), - "value" => Value::test_string("a"), - })), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - dataframe_command(engine_state, stack, call, value) - } else { - expression_command(call, value) - } - } -} - -fn dataframe_command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: Value, -) -> Result { - let rows: Option = call.get_flag(engine_state, stack, "rows")?; - let tail: bool = call.has_flag(engine_state, stack, "tail")?; - - let df = NuDataFrame::try_from_value(input)?; - - let values = if tail { - df.tail(rows, call.head)? - } else { - // if rows is specified, return those rows, otherwise return everything - if rows.is_some() { - df.head(rows, call.head)? - } else { - df.head(Some(df.height()), call.head)? - } - }; - - let value = Value::list(values, call.head); - - Ok(PipelineData::Value(value, None)) -} -fn expression_command(call: &Call, input: Value) -> Result { - let expr = NuExpression::try_from_value(input)?; - let value = expr.to_value(call.head)?; - - Ok(PipelineData::Value(value, None)) -} - -#[cfg(test)] -mod test { - use super::super::super::expressions::ExprCol; - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples_dataframe_input() { - test_dataframe(vec![Box::new(ToNu {})]) - } - - #[test] - fn test_examples_expression_input() { - test_dataframe(vec![Box::new(ToNu {}), Box::new(ExprCol {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_parquet.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_parquet.rs deleted file mode 100644 index ce6419a9ac..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_parquet.rs +++ /dev/null @@ -1,79 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::prelude::ParquetWriter; -use std::{fs::File, path::PathBuf}; - -#[derive(Clone)] -pub struct ToParquet; - -impl Command for ToParquet { - fn name(&self) -> &str { - "dfr to-parquet" - } - - fn usage(&self) -> &str { - "Saves dataframe to parquet file." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("file", SyntaxShape::Filepath, "file path to save dataframe") - .input_output_type(Type::Custom("dataframe".into()), Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Saves dataframe to parquet file", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-parquet test.parquet", - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let file_name: Spanned = call.req(engine_state, stack, 0)?; - - let mut df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError { - error: "Error with file name".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - ParquetWriter::new(file) - .finish(df.as_mut()) - .map_err(|e| ShellError::GenericError { - error: "Error saving file".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span); - - Ok(PipelineData::Value( - Value::list(vec![file_value], call.head), - None, - )) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/alias.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/alias.rs deleted file mode 100644 index 9d36100276..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/alias.rs +++ /dev/null @@ -1,86 +0,0 @@ -use crate::dataframe::values::NuExpression; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ExprAlias; - -impl Command for ExprAlias { - fn name(&self) -> &str { - "dfr as" - } - - fn usage(&self) -> &str { - "Creates an alias expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "Alias name", - SyntaxShape::String, - "Alias name for the expression", - ) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Creates and alias expression", - example: "dfr col a | dfr as new_a | dfr into-nu", - result: { - let record = Value::test_record(record! { - "expr" => Value::test_record(record! { - "expr" => Value::test_string("column"), - "value" => Value::test_string("a"), - }), - "alias" => Value::test_string("new_a"), - }); - - Some(record) - }, - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["aka", "abbr", "otherwise"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let alias: String = call.req(engine_state, stack, 0)?; - - let expr = NuExpression::try_from_pipeline(input, call.head)?; - let expr: NuExpression = expr.into_polars().alias(alias.as_str()).into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::eager::ToNu; - use crate::dataframe::expressions::ExprCol; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(ExprAlias {}), - Box::new(ExprCol {}), - Box::new(ToNu {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/arg_where.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/arg_where.rs deleted file mode 100644 index 49c13c3f44..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/arg_where.rs +++ /dev/null @@ -1,78 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -use polars::prelude::arg_where; - -#[derive(Clone)] -pub struct ExprArgWhere; - -impl Command for ExprArgWhere { - fn name(&self) -> &str { - "dfr arg-where" - } - - fn usage(&self) -> &str { - "Creates an expression that returns the arguments where expression is true." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("column name", SyntaxShape::Any, "Expression to evaluate") - .input_output_type(Type::Any, Type::Custom("expression".into())) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Return a dataframe where the value match the expression", - example: "let df = ([[a b]; [one 1] [two 2] [three 3]] | dfr into-df); - $df | dfr select (dfr arg-where ((dfr col b) >= 2) | dfr as b_arg)", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "b_arg".to_string(), - vec![Value::test_int(1), Value::test_int(2)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["condition", "match", "if"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - let value: Value = call.req(engine_state, stack, 0)?; - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = arg_where(expr.into_polars()).into(); - - Ok(PipelineData::Value(expr.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::expressions::ExprAlias; - use crate::dataframe::lazy::LazySelect; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(ExprArgWhere {}), - Box::new(ExprAlias {}), - Box::new(LazySelect {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/col.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/col.rs deleted file mode 100644 index 1520ef995d..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/col.rs +++ /dev/null @@ -1,68 +0,0 @@ -use crate::dataframe::values::NuExpression; -use nu_engine::command_prelude::*; - -use polars::prelude::col; - -#[derive(Clone)] -pub struct ExprCol; - -impl Command for ExprCol { - fn name(&self) -> &str { - "dfr col" - } - - fn usage(&self) -> &str { - "Creates a named column expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "column name", - SyntaxShape::String, - "Name of column to be used", - ) - .input_output_type(Type::Any, Type::Custom("expression".into())) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Creates a named column expression and converts it to a nu object", - example: "dfr col a | dfr into-nu", - result: Some(Value::test_record(record! { - "expr" => Value::test_string("column"), - "value" => Value::test_string("a"), - })), - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["create"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - let name: String = call.req(engine_state, stack, 0)?; - let expr: NuExpression = col(name.as_str()).into(); - - Ok(PipelineData::Value(expr.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::eager::ToNu; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ExprCol {}), Box::new(ToNu {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/concat_str.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/concat_str.rs deleted file mode 100644 index 28f9bbda71..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/concat_str.rs +++ /dev/null @@ -1,108 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -use polars::prelude::concat_str; - -#[derive(Clone)] -pub struct ExprConcatStr; - -impl Command for ExprConcatStr { - fn name(&self) -> &str { - "dfr concat-str" - } - - fn usage(&self) -> &str { - "Creates a concat string expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "separator", - SyntaxShape::String, - "Separator used during the concatenation", - ) - .required( - "concat expressions", - SyntaxShape::List(Box::new(SyntaxShape::Any)), - "Expression(s) that define the string concatenation", - ) - .input_output_type(Type::Any, Type::Custom("expression".into())) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Creates a concat string expression", - example: r#"let df = ([[a b c]; [one two 1] [three four 2]] | dfr into-df); - $df | dfr with-column ((dfr concat-str "-" [(dfr col a) (dfr col b) ((dfr col c) * 2)]) | dfr as concat)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("three")], - ), - Column::new( - "b".to_string(), - vec![Value::test_string("two"), Value::test_string("four")], - ), - Column::new( - "c".to_string(), - vec![Value::test_int(1), Value::test_int(2)], - ), - Column::new( - "concat".to_string(), - vec![ - Value::test_string("one-two-2"), - Value::test_string("three-four-4"), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["join", "connect", "update"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - let separator: String = call.req(engine_state, stack, 0)?; - let value: Value = call.req(engine_state, stack, 1)?; - - let expressions = NuExpression::extract_exprs(value)?; - let expr: NuExpression = concat_str(expressions, &separator, false).into(); - - Ok(PipelineData::Value(expr.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::eager::WithColumn; - use crate::dataframe::expressions::alias::ExprAlias; - use crate::dataframe::expressions::col::ExprCol; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(ExprConcatStr {}), - Box::new(ExprAlias {}), - Box::new(ExprCol {}), - Box::new(WithColumn {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/datepart.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/datepart.rs deleted file mode 100644 index 60913c0dc6..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/datepart.rs +++ /dev/null @@ -1,170 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use chrono::{DateTime, FixedOffset}; -use nu_engine::command_prelude::*; - -use polars::{ - datatypes::{DataType, TimeUnit}, - prelude::NamedFrom, - series::Series, -}; - -#[derive(Clone)] -pub struct ExprDatePart; - -impl Command for ExprDatePart { - fn name(&self) -> &str { - "dfr datepart" - } - - fn usage(&self) -> &str { - "Creates an expression for capturing the specified datepart in a column." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "Datepart name", - SyntaxShape::String, - "Part of the date to capture. Possible values are year, quarter, month, week, weekday, day, hour, minute, second, millisecond, microsecond, nanosecond", - ) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - let dt = DateTime::::parse_from_str( - "2021-12-30T01:02:03.123456789 +0000", - "%Y-%m-%dT%H:%M:%S.%9f %z", - ) - .expect("date calculation should not fail in test"); - vec![ - Example { - description: "Creates an expression to capture the year date part", - example: r#"[["2021-12-30T01:02:03.123456789"]] | dfr into-df | dfr as-datetime "%Y-%m-%dT%H:%M:%S.%9f" | dfr with-column [(dfr col datetime | dfr datepart year | dfr as datetime_year )]"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("datetime".to_string(), vec![Value::test_date(dt)]), - Column::new("datetime_year".to_string(), vec![Value::test_int(2021)]), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates an expression to capture multiple date parts", - example: r#"[["2021-12-30T01:02:03.123456789"]] | dfr into-df | dfr as-datetime "%Y-%m-%dT%H:%M:%S.%9f" | - dfr with-column [ (dfr col datetime | dfr datepart year | dfr as datetime_year ), - (dfr col datetime | dfr datepart month | dfr as datetime_month ), - (dfr col datetime | dfr datepart day | dfr as datetime_day ), - (dfr col datetime | dfr datepart hour | dfr as datetime_hour ), - (dfr col datetime | dfr datepart minute | dfr as datetime_minute ), - (dfr col datetime | dfr datepart second | dfr as datetime_second ), - (dfr col datetime | dfr datepart nanosecond | dfr as datetime_ns ) ]"#, - result: Some( - NuDataFrame::try_from_series( - vec![ - Series::new("datetime", &[dt.timestamp_nanos_opt()]) - .cast(&DataType::Datetime(TimeUnit::Nanoseconds, None)) - .expect("Error casting to datetime type"), - Series::new("datetime_year", &[2021_i64]), // i32 was coerced to i64 - Series::new("datetime_month", &[12_i8]), - Series::new("datetime_day", &[30_i8]), - Series::new("datetime_hour", &[1_i8]), - Series::new("datetime_minute", &[2_i8]), - Series::new("datetime_second", &[3_i8]), - Series::new("datetime_ns", &[123456789_i64]), // i32 was coerced to i64 - ], - Span::test_data(), - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn search_terms(&self) -> Vec<&str> { - vec![ - "year", - "month", - "week", - "weekday", - "quarter", - "day", - "hour", - "minute", - "second", - "millisecond", - "microsecond", - "nanosecond", - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let part: Spanned = call.req(engine_state, stack, 0)?; - - let expr = NuExpression::try_from_pipeline(input, call.head)?; - let expr_dt = expr.into_polars().dt(); - let expr = match part.item.as_str() { - "year" => expr_dt.year(), - "quarter" => expr_dt.quarter(), - "month" => expr_dt.month(), - "week" => expr_dt.week(), - "day" => expr_dt.day(), - "hour" => expr_dt.hour(), - "minute" => expr_dt.minute(), - "second" => expr_dt.second(), - "millisecond" => expr_dt.millisecond(), - "microsecond" => expr_dt.microsecond(), - "nanosecond" => expr_dt.nanosecond(), - _ => { - return Err(ShellError::UnsupportedInput { - msg: format!("{} is not a valid datepart, expected one of year, month, day, hour, minute, second, millisecond, microsecond, nanosecond", part.item), - input: "value originates from here".to_string(), - msg_span: call.head, - input_span: part.span, - }); - } - }.into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::eager::ToNu; - use crate::dataframe::eager::WithColumn; - use crate::dataframe::expressions::ExprAlias; - use crate::dataframe::expressions::ExprCol; - use crate::dataframe::series::AsDateTime; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(ExprDatePart {}), - Box::new(ExprCol {}), - Box::new(ToNu {}), - Box::new(AsDateTime {}), - Box::new(WithColumn {}), - Box::new(ExprAlias {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs deleted file mode 100644 index 4cc56e030b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs +++ /dev/null @@ -1,736 +0,0 @@ -/// Definition of multiple Expression commands using a macro rule -/// All of these expressions have an identical body and only require -/// to have a change in the name, description and expression function -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -// The structs defined in this file are structs that form part of other commands -// since they share a similar name -macro_rules! expr_command { - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let expr = NuExpression::try_from_pipeline(input, call.head)?; - let expr: NuExpression = expr.into_polars().$func().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new($command {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]) - } - } - }; - - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident, $ddof: expr) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let expr = NuExpression::try_from_pipeline(input, call.head)?; - let expr: NuExpression = expr.into_polars().$func($ddof).into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new($command {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]) - } - } - }; -} - -// The structs defined in this file are structs that form part of other commands -// since they share a similar name -macro_rules! lazy_expr_command { - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let lazy = NuLazyFrame::try_from_value(value)?; - let lazy = NuLazyFrame::new( - lazy.from_eager, - lazy.into_polars() - .$func() - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - help: None, - span: None, - inner: vec![], - })?, - ); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().$func().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::{ - build_test_engine_state, test_dataframe_example, - }; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - // the first example should be a for the dataframe case - let example = &$command.examples()[0]; - let mut engine_state = build_test_engine_state(vec![Box::new($command {})]); - test_dataframe_example(&mut engine_state, &example) - } - - #[test] - fn test_examples_expressions() { - // the second example should be a for the dataframe case - let example = &$command.examples()[1]; - let mut engine_state = build_test_engine_state(vec![ - Box::new($command {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &example) - } - } - }; - - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident, $ddof: expr) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let lazy = NuLazyFrame::try_from_value(value)?; - let lazy = NuLazyFrame::new( - lazy.from_eager, - lazy.into_polars() - .$func($ddof) - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - help: None, - span: None, - inner: vec![], - })?, - ); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().$func($ddof).into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::{ - build_test_engine_state, test_dataframe_example, - }; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - // the first example should be a for the dataframe case - let example = &$command.examples()[0]; - let mut engine_state = build_test_engine_state(vec![Box::new($command {})]); - test_dataframe_example(&mut engine_state, &example) - } - - #[test] - fn test_examples_expressions() { - // the second example should be a for the dataframe case - let example = &$command.examples()[1]; - let mut engine_state = build_test_engine_state(vec![ - Box::new($command {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &example) - } - } - }; -} - -// ExprList command -// Expands to a command definition for a list expression -expr_command!( - ExprList, - "dfr implode", - "Aggregates a group to a Series.", - vec![Example { - description: "", - example: "", - result: None, - }], - implode, - test_implode -); - -// ExprAggGroups command -// Expands to a command definition for a agg groups expression -expr_command!( - ExprAggGroups, - "dfr agg-groups", - "Creates an agg_groups expression.", - vec![Example { - description: "", - example: "", - result: None, - }], - agg_groups, - test_groups -); - -// ExprCount command -// Expands to a command definition for a count expression -expr_command!( - ExprCount, - "dfr count", - "Creates a count expression.", - vec![Example { - description: "", - example: "", - result: None, - }], - count, - test_count -); - -// ExprNot command -// Expands to a command definition for a not expression -expr_command!( - ExprNot, - "dfr expr-not", - "Creates a not expression.", - vec![Example { - description: "Creates a not expression", - example: "(dfr col a) > 2) | dfr expr-not", - result: None, - },], - not, - test_not -); - -// ExprMax command -// Expands to a command definition for max aggregation -lazy_expr_command!( - ExprMax, - "dfr max", - "Creates a max expression or aggregates columns to their max value.", - vec![ - Example { - description: "Max value from columns in a dataframe", - example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr max", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_int(6)],), - Column::new("b".to_string(), vec![Value::test_int(4)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Max aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 4] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr max)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(4), Value::test_int(1)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ], - max, - test_max -); - -// ExprMin command -// Expands to a command definition for min aggregation -lazy_expr_command!( - ExprMin, - "dfr min", - "Creates a min expression or aggregates columns to their min value.", - vec![ - Example { - description: "Min value from columns in a dataframe", - example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr min", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_int(1)],), - Column::new("b".to_string(), vec![Value::test_int(1)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Min aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 4] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr min)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(1)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ], - min, - test_min -); - -// ExprSum command -// Expands to a command definition for sum aggregation -lazy_expr_command!( - ExprSum, - "dfr sum", - "Creates a sum expression for an aggregation or aggregates columns to their sum value.", - vec![ - Example { - description: "Sums all columns in a dataframe", - example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr sum", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_int(11)],), - Column::new("b".to_string(), vec![Value::test_int(7)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Sum aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 4] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr sum)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(6), Value::test_int(1)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ], - sum, - test_sum -); - -// ExprMean command -// Expands to a command definition for mean aggregation -lazy_expr_command!( - ExprMean, - "dfr mean", - "Creates a mean expression for an aggregation or aggregates columns to their mean value.", - vec![ - Example { - description: "Mean value from columns in a dataframe", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr mean", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_float(4.0)],), - Column::new("b".to_string(), vec![Value::test_float(2.0)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Mean aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 4] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr mean)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_float(3.0), Value::test_float(1.0)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ], - mean, - test_mean -); - -// ExprMedian command -// Expands to a command definition for median aggregation -expr_command!( - ExprMedian, - "dfr median", - "Creates a median expression for an aggregation.", - vec![Example { - description: "Median aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 4] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr median)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_float(3.0), Value::test_float(1.0)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - },], - median, - test_median -); - -// ExprStd command -// Expands to a command definition for std aggregation -lazy_expr_command!( - ExprStd, - "dfr std", - "Creates a std expression for an aggregation of std value from columns in a dataframe.", - vec![ - Example { - description: "Std value from columns in a dataframe", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr std", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_float(2.0)],), - Column::new("b".to_string(), vec![Value::test_float(0.0)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Std aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 2] [two 1] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr std)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_float(0.0), Value::test_float(0.0)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ], - std, - test_std, - 1 -); - -// ExprVar command -// Expands to a command definition for var aggregation -lazy_expr_command!( - ExprVar, - "dfr var", - "Create a var expression for an aggregation.", - vec![ - Example { - description: - "Var value from columns in a dataframe or aggregates columns to their var value", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr var", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_float(4.0)],), - Column::new("b".to_string(), vec![Value::test_float(0.0)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Var aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 2] [two 1] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr var)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_float(0.0), Value::test_float(0.0)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ], - var, - test_var, - 1 -); diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/is_in.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/is_in.rs deleted file mode 100644 index 1579ba0e20..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/is_in.rs +++ /dev/null @@ -1,116 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -use polars::prelude::{lit, DataType}; - -#[derive(Clone)] -pub struct ExprIsIn; - -impl Command for ExprIsIn { - fn name(&self) -> &str { - "dfr is-in" - } - - fn usage(&self) -> &str { - "Creates an is-in expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "list", - SyntaxShape::List(Box::new(SyntaxShape::Any)), - "List to check if values are in", - ) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Creates a is-in expression", - example: r#"let df = ([[a b]; [one 1] [two 2] [three 3]] | dfr into-df); - $df | dfr with-column (dfr col a | dfr is-in [one two] | dfr as a_in)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![ - Value::test_string("one"), - Value::test_string("two"), - Value::test_string("three"), - ], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(1), Value::test_int(2), Value::test_int(3)], - ), - Column::new( - "a_in".to_string(), - vec![ - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(false), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["check", "contained", "is-contain", "match"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let list: Vec = call.req(engine_state, stack, 0)?; - let expr = NuExpression::try_from_pipeline(input, call.head)?; - - let values = - NuDataFrame::try_from_columns(vec![Column::new("list".to_string(), list)], None)?; - let list = values.as_series(call.head)?; - - if matches!(list.dtype(), DataType::Object(..)) { - return Err(ShellError::IncompatibleParametersSingle { - msg: "Cannot use a mixed list as argument".into(), - span: call.head, - }); - } - - let expr: NuExpression = expr.into_polars().is_in(lit(list)).into(); - Ok(PipelineData::Value(expr.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::eager::WithColumn; - use crate::dataframe::expressions::alias::ExprAlias; - use crate::dataframe::expressions::col::ExprCol; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(ExprIsIn {}), - Box::new(ExprAlias {}), - Box::new(ExprCol {}), - Box::new(WithColumn {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/lit.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/lit.rs deleted file mode 100644 index 8610a59048..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/lit.rs +++ /dev/null @@ -1,69 +0,0 @@ -use crate::dataframe::values::NuExpression; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ExprLit; - -impl Command for ExprLit { - fn name(&self) -> &str { - "dfr lit" - } - - fn usage(&self) -> &str { - "Creates a literal expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "literal", - SyntaxShape::Any, - "literal to construct the expression", - ) - .input_output_type(Type::Any, Type::Custom("expression".into())) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Created a literal expression and converts it to a nu object", - example: "dfr lit 2 | dfr into-nu", - result: Some(Value::test_record(record! { - "expr" => Value::test_string("literal"), - "value" => Value::test_string("2"), - })), - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["string", "literal", "expression"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - let literal: Value = call.req(engine_state, stack, 0)?; - - let expr = NuExpression::try_from_value(literal)?; - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::eager::ToNu; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ExprLit {}), Box::new(ToNu {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/mod.rs deleted file mode 100644 index 4ba70d900d..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/mod.rs +++ /dev/null @@ -1,62 +0,0 @@ -mod alias; -mod arg_where; -mod col; -mod concat_str; -mod datepart; -mod expressions_macro; -mod is_in; -mod lit; -mod otherwise; -mod quantile; -mod when; - -use nu_protocol::engine::StateWorkingSet; - -pub(crate) use crate::dataframe::expressions::alias::ExprAlias; -use crate::dataframe::expressions::arg_where::ExprArgWhere; -pub(super) use crate::dataframe::expressions::col::ExprCol; -pub(super) use crate::dataframe::expressions::concat_str::ExprConcatStr; -pub(crate) use crate::dataframe::expressions::datepart::ExprDatePart; -pub(crate) use crate::dataframe::expressions::expressions_macro::*; -pub(super) use crate::dataframe::expressions::is_in::ExprIsIn; -pub(super) use crate::dataframe::expressions::lit::ExprLit; -pub(super) use crate::dataframe::expressions::otherwise::ExprOtherwise; -pub(super) use crate::dataframe::expressions::quantile::ExprQuantile; -pub(super) use crate::dataframe::expressions::when::ExprWhen; - -pub fn add_expressions(working_set: &mut StateWorkingSet) { - macro_rules! bind_command { - ( $command:expr ) => { - working_set.add_decl(Box::new($command)); - }; - ( $( $command:expr ),* ) => { - $( working_set.add_decl(Box::new($command)); )* - }; - } - - // Dataframe commands - bind_command!( - ExprAlias, - ExprArgWhere, - ExprCol, - ExprConcatStr, - ExprCount, - ExprLit, - ExprWhen, - ExprOtherwise, - ExprQuantile, - ExprList, - ExprAggGroups, - ExprCount, - ExprIsIn, - ExprNot, - ExprMax, - ExprMin, - ExprSum, - ExprMean, - ExprMedian, - ExprStd, - ExprVar, - ExprDatePart - ); -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs deleted file mode 100644 index eb97c575b7..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs +++ /dev/null @@ -1,126 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuWhen}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ExprOtherwise; - -impl Command for ExprOtherwise { - fn name(&self) -> &str { - "dfr otherwise" - } - - fn usage(&self) -> &str { - "Completes a when expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "otherwise expression", - SyntaxShape::Any, - "expression to apply when no when predicate matches", - ) - .input_output_type(Type::Any, Type::Custom("expression".into())) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create a when conditions", - example: "dfr when ((dfr col a) > 2) 4 | dfr otherwise 5", - result: None, - }, - Example { - description: "Create a when conditions", - example: - "dfr when ((dfr col a) > 2) 4 | dfr when ((dfr col a) < 0) 6 | dfr otherwise 0", - result: None, - }, - Example { - description: "Create a new column for the dataframe", - example: r#"[[a b]; [6 2] [1 4] [4 1]] - | dfr into-lazy - | dfr with-column ( - dfr when ((dfr col a) > 2) 4 | dfr otherwise 5 | dfr as c - ) - | dfr with-column ( - dfr when ((dfr col a) > 5) 10 | dfr when ((dfr col a) < 2) 6 | dfr otherwise 0 | dfr as d - ) - | dfr collect"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)], - ), - Column::new( - "c".to_string(), - vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)], - ), - Column::new( - "d".to_string(), - vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["condition", "else"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let otherwise_predicate: Value = call.req(engine_state, stack, 0)?; - let otherwise_predicate = NuExpression::try_from_value(otherwise_predicate)?; - - let value = input.into_value(call.head)?; - let complete: NuExpression = match NuWhen::try_from_value(value)? { - NuWhen::Then(then) => then.otherwise(otherwise_predicate.into_polars()).into(), - NuWhen::ChainedThen(chained_when) => chained_when - .otherwise(otherwise_predicate.into_polars()) - .into(), - }; - - Ok(PipelineData::Value(complete.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use crate::dataframe::eager::{ToNu, WithColumn}; - use crate::dataframe::expressions::when::ExprWhen; - use crate::dataframe::expressions::{ExprAlias, ExprCol}; - - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(WithColumn {}), - Box::new(ExprCol {}), - Box::new(ExprAlias {}), - Box::new(ExprWhen {}), - Box::new(ExprOtherwise {}), - Box::new(ToNu {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs deleted file mode 100644 index aaa1029ee9..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs +++ /dev/null @@ -1,101 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -use polars::prelude::{lit, QuantileInterpolOptions}; - -#[derive(Clone)] -pub struct ExprQuantile; - -impl Command for ExprQuantile { - fn name(&self) -> &str { - "dfr quantile" - } - - fn usage(&self) -> &str { - "Aggregates the columns to the selected quantile." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "quantile", - SyntaxShape::Number, - "quantile value for quantile operation", - ) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Quantile aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 4] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr quantile 0.5)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_float(4.0), Value::test_float(1.0)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["statistics", "percentile", "distribution"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - let quantile: f64 = call.req(engine_state, stack, 0)?; - - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr - .into_polars() - .quantile(lit(quantile), QuantileInterpolOptions::default()) - .into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(ExprQuantile {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs deleted file mode 100644 index 5a6aad2de7..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs +++ /dev/null @@ -1,147 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuWhen}; -use nu_engine::command_prelude::*; - -use polars::prelude::when; - -#[derive(Clone)] -pub struct ExprWhen; - -impl Command for ExprWhen { - fn name(&self) -> &str { - "dfr when" - } - - fn usage(&self) -> &str { - "Creates and modifies a when expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "when expression", - SyntaxShape::Any, - "when expression used for matching", - ) - .required( - "then expression", - SyntaxShape::Any, - "expression that will be applied when predicate is true", - ) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create a when conditions", - example: "dfr when ((dfr col a) > 2) 4", - result: None, - }, - Example { - description: "Create a when conditions", - example: "dfr when ((dfr col a) > 2) 4 | dfr when ((dfr col a) < 0) 6", - result: None, - }, - Example { - description: "Create a new column for the dataframe", - example: r#"[[a b]; [6 2] [1 4] [4 1]] - | dfr into-lazy - | dfr with-column ( - dfr when ((dfr col a) > 2) 4 | dfr otherwise 5 | dfr as c - ) - | dfr with-column ( - dfr when ((dfr col a) > 5) 10 | dfr when ((dfr col a) < 2) 6 | dfr otherwise 0 | dfr as d - ) - | dfr collect"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)], - ), - Column::new( - "c".to_string(), - vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)], - ), - Column::new( - "d".to_string(), - vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["condition", "match", "if", "else"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let when_predicate: Value = call.req(engine_state, stack, 0)?; - let when_predicate = NuExpression::try_from_value(when_predicate)?; - - let then_predicate: Value = call.req(engine_state, stack, 1)?; - let then_predicate = NuExpression::try_from_value(then_predicate)?; - - let value = input.into_value(call.head)?; - let when_then: NuWhen = match value { - Value::Nothing { .. } => when(when_predicate.into_polars()) - .then(then_predicate.into_polars()) - .into(), - v => match NuWhen::try_from_value(v)? { - NuWhen::Then(when_then) => when_then - .when(when_predicate.into_polars()) - .then(then_predicate.into_polars()) - .into(), - NuWhen::ChainedThen(when_then_then) => when_then_then - .when(when_predicate.into_polars()) - .then(then_predicate.into_polars()) - .into(), - }, - }; - - Ok(PipelineData::Value(when_then.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use crate::dataframe::eager::{ToNu, WithColumn}; - use crate::dataframe::expressions::otherwise::ExprOtherwise; - use crate::dataframe::expressions::{ExprAlias, ExprCol}; - - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(WithColumn {}), - Box::new(ExprCol {}), - Box::new(ExprAlias {}), - Box::new(ExprWhen {}), - Box::new(ExprOtherwise {}), - Box::new(ToNu {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs deleted file mode 100644 index 715c3d156b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs +++ /dev/null @@ -1,216 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame, NuLazyGroupBy}; -use nu_engine::command_prelude::*; - -use polars::{datatypes::DataType, prelude::Expr}; - -#[derive(Clone)] -pub struct LazyAggregate; - -impl Command for LazyAggregate { - fn name(&self) -> &str { - "dfr agg" - } - - fn usage(&self) -> &str { - "Performs a series of aggregations from a group-by." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest( - "Group-by expressions", - SyntaxShape::Any, - "Expression(s) that define the aggregations to be applied", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Group by and perform an aggregation", - example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]] - | dfr into-df - | dfr group-by a - | dfr agg [ - (dfr col b | dfr min | dfr as "b_min") - (dfr col b | dfr max | dfr as "b_max") - (dfr col b | dfr sum | dfr as "b_sum") - ]"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(2)], - ), - Column::new( - "b_min".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - Column::new( - "b_max".to_string(), - vec![Value::test_int(4), Value::test_int(6)], - ), - Column::new( - "b_sum".to_string(), - vec![Value::test_int(6), Value::test_int(10)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Group by and perform an aggregation", - example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]] - | dfr into-lazy - | dfr group-by a - | dfr agg [ - (dfr col b | dfr min | dfr as "b_min") - (dfr col b | dfr max | dfr as "b_max") - (dfr col b | dfr sum | dfr as "b_sum") - ] - | dfr collect"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(2)], - ), - Column::new( - "b_min".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - Column::new( - "b_max".to_string(), - vec![Value::test_int(4), Value::test_int(6)], - ), - Column::new( - "b_sum".to_string(), - vec![Value::test_int(6), Value::test_int(10)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let vals: Vec = call.rest(engine_state, stack, 0)?; - let value = Value::list(vals, call.head); - let expressions = NuExpression::extract_exprs(value)?; - - let group_by = NuLazyGroupBy::try_from_pipeline(input, call.head)?; - - if let Some(schema) = &group_by.schema { - for expr in expressions.iter() { - if let Some(name) = get_col_name(expr) { - let dtype = schema.get(name.as_str()); - - if matches!(dtype, Some(DataType::Object(..))) { - return Err(ShellError::GenericError { - error: "Object type column not supported for aggregation".into(), - msg: format!("Column '{name}' is type Object"), - span: Some(call.head), - help: Some("Aggregations cannot be performed on Object type columns. Use dtype command to check column types".into()), - inner: vec![], - }); - } - } - } - } - - let lazy = NuLazyFrame { - from_eager: group_by.from_eager, - lazy: Some(group_by.into_polars().agg(&expressions)), - schema: None, - }; - - let res = lazy.into_value(call.head)?; - Ok(PipelineData::Value(res, None)) - } -} - -fn get_col_name(expr: &Expr) -> Option { - match expr { - Expr::Column(column) => Some(column.to_string()), - Expr::Agg(agg) => match agg { - polars::prelude::AggExpr::Min { input: e, .. } - | polars::prelude::AggExpr::Max { input: e, .. } - | polars::prelude::AggExpr::Median(e) - | polars::prelude::AggExpr::NUnique(e) - | polars::prelude::AggExpr::First(e) - | polars::prelude::AggExpr::Last(e) - | polars::prelude::AggExpr::Mean(e) - | polars::prelude::AggExpr::Implode(e) - | polars::prelude::AggExpr::Count(e, _) - | polars::prelude::AggExpr::Sum(e) - | polars::prelude::AggExpr::AggGroups(e) - | polars::prelude::AggExpr::Std(e, _) - | polars::prelude::AggExpr::Var(e, _) => get_col_name(e.as_ref()), - polars::prelude::AggExpr::Quantile { expr, .. } => get_col_name(expr.as_ref()), - }, - Expr::Filter { input: expr, .. } - | Expr::Slice { input: expr, .. } - | Expr::Cast { expr, .. } - | Expr::Sort { expr, .. } - | Expr::Gather { expr, .. } - | Expr::SortBy { expr, .. } - | Expr::Exclude(expr, _) - | Expr::Alias(expr, _) - | Expr::KeepName(expr) - | Expr::Explode(expr) => get_col_name(expr.as_ref()), - Expr::Ternary { .. } - | Expr::AnonymousFunction { .. } - | Expr::Function { .. } - | Expr::Columns(_) - | Expr::DtypeColumn(_) - | Expr::Literal(_) - | Expr::BinaryExpr { .. } - | Expr::Window { .. } - | Expr::Wildcard - | Expr::RenameAlias { .. } - | Expr::Len - | Expr::Nth(_) - | Expr::SubPlan(_, _) - | Expr::Selector(_) => None, - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::expressions::{ExprAlias, ExprMax, ExprMin, ExprSum}; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - Box::new(ExprAlias {}), - Box::new(ExprMin {}), - Box::new(ExprMax {}), - Box::new(ExprSum {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/collect.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/collect.rs deleted file mode 100644 index c27591cc1d..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/collect.rs +++ /dev/null @@ -1,73 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyCollect; - -impl Command for LazyCollect { - fn name(&self) -> &str { - "dfr collect" - } - - fn usage(&self) -> &str { - "Collect lazy dataframe into eager dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "drop duplicates", - example: "[[a b]; [1 2] [3 4]] | dfr into-lazy | dfr collect", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let eager = lazy.collect(call.head)?; - let value = Value::custom(Box::new(eager), call.head); - - Ok(PipelineData::Value(value, None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyCollect {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs deleted file mode 100644 index a027e84d36..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs +++ /dev/null @@ -1,153 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyExplode; - -impl Command for LazyExplode { - fn name(&self) -> &str { - "dfr explode" - } - - fn usage(&self) -> &str { - "Explodes a dataframe or creates a explode expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest( - "columns", - SyntaxShape::String, - "columns to explode, only applicable for dataframes", - ) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Explode the specified dataframe", - example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr explode hobbies | dfr collect", - result: Some( - NuDataFrame::try_from_columns(vec![ - Column::new( - "id".to_string(), - vec![ - Value::test_int(1), - Value::test_int(1), - Value::test_int(2), - Value::test_int(2), - ]), - Column::new( - "name".to_string(), - vec![ - Value::test_string("Mercy"), - Value::test_string("Mercy"), - Value::test_string("Bob"), - Value::test_string("Bob"), - ]), - Column::new( - "hobbies".to_string(), - vec![ - Value::test_string("Cycling"), - Value::test_string("Knitting"), - Value::test_string("Skiing"), - Value::test_string("Football"), - ]), - ], None).expect("simple df for test should not fail") - .into_value(Span::test_data()), - ) - }, - Example { - description: "Select a column and explode the values", - example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr select (dfr col hobbies | dfr explode)", - result: Some( - NuDataFrame::try_from_columns(vec![ - Column::new( - "hobbies".to_string(), - vec![ - Value::test_string("Cycling"), - Value::test_string("Knitting"), - Value::test_string("Skiing"), - Value::test_string("Football"), - ]), - ], None).expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - explode(call, input) - } -} - -pub(crate) fn explode(call: &Call, input: PipelineData) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let df = NuLazyFrame::try_from_value(value)?; - let columns: Vec = call - .positional_iter() - .filter_map(|e| e.as_string()) - .collect(); - - let exploded = df - .into_polars() - .explode(columns.iter().map(AsRef::as_ref).collect::>()); - - Ok(PipelineData::Value( - NuLazyFrame::from(exploded).into_value(call.head)?, - None, - )) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().explode().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example}; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(LazyExplode {})]); - test_dataframe_example(&mut engine_state, &LazyExplode.examples()[0]); - } - - #[ignore] - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(LazyExplode {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &LazyExplode.examples()[1]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/fetch.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/fetch.rs deleted file mode 100644 index 6ba75aa970..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/fetch.rs +++ /dev/null @@ -1,92 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyFetch; - -impl Command for LazyFetch { - fn name(&self) -> &str { - "dfr fetch" - } - - fn usage(&self) -> &str { - "Collects the lazyframe to the selected rows." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "rows", - SyntaxShape::Int, - "number of rows to be fetched from lazyframe", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Fetch a rows from the dataframe", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr fetch 2", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(6), Value::test_int(4)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(2)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let rows: i64 = call.req(engine_state, stack, 0)?; - - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let eager: NuDataFrame = lazy - .into_polars() - .fetch(rows as usize) - .map_err(|e| ShellError::GenericError { - error: "Error fetching rows".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(PipelineData::Value( - NuDataFrame::into_value(eager, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyFetch {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs deleted file mode 100644 index 4c75f1d9a3..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs +++ /dev/null @@ -1,143 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyFillNA; - -impl Command for LazyFillNA { - fn name(&self) -> &str { - "dfr fill-nan" - } - - fn usage(&self) -> &str { - "Replaces NaN values with the given expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "fill", - SyntaxShape::Any, - "Expression to use to fill the NAN values", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Fills the NaN values with 0", - example: "[1 2 NaN 3 NaN] | dfr into-df | dfr fill-nan 0", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_int(1), - Value::test_int(2), - Value::test_int(0), - Value::test_int(3), - Value::test_int(0), - ], - )], - None, - ) - .expect("Df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Fills the NaN values of a whole dataframe", - example: "[[a b]; [0.2 1] [0.1 NaN]] | dfr into-df | dfr fill-nan 0", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_float(0.2), Value::test_float(0.1)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(1), Value::test_int(0)], - ), - ], - None, - ) - .expect("Df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let fill: Value = call.req(engine_state, stack, 0)?; - let value = input.into_value(call.head)?; - - if NuExpression::can_downcast(&value) { - let expr = NuExpression::try_from_value(value)?; - let fill = NuExpression::try_from_value(fill)?.into_polars(); - let expr: NuExpression = expr.into_polars().fill_nan(fill).into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } else { - let val_span = value.span(); - let frame = NuDataFrame::try_from_value(value)?; - let columns = frame.columns(val_span)?; - let dataframe = columns - .into_iter() - .map(|column| { - let column_name = column.name().to_string(); - let values = column - .into_iter() - .map(|value| { - let span = value.span(); - match value { - Value::Float { val, .. } => { - if val.is_nan() { - fill.clone() - } else { - value - } - } - Value::List { vals, .. } => { - NuDataFrame::fill_list_nan(vals, span, fill.clone()) - } - _ => value, - } - }) - .collect::>(); - Column::new(column_name, values) - }) - .collect::>(); - Ok(PipelineData::Value( - NuDataFrame::try_from_columns(dataframe, None)?.into_value(call.head), - None, - )) - } - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyFillNA {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs deleted file mode 100644 index 88be2a9e88..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs +++ /dev/null @@ -1,93 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyFillNull; - -impl Command for LazyFillNull { - fn name(&self) -> &str { - "dfr fill-null" - } - - fn usage(&self) -> &str { - "Replaces NULL values with the given expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "fill", - SyntaxShape::Any, - "Expression to use to fill the null values", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Fills the null values by 0", - example: "[1 2 2 3 3] | dfr into-df | dfr shift 2 | dfr fill-null 0", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_int(0), - Value::test_int(0), - Value::test_int(1), - Value::test_int(2), - Value::test_int(2), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let fill: Value = call.req(engine_state, stack, 0)?; - let value = input.into_value(call.head)?; - - if NuExpression::can_downcast(&value) { - let expr = NuExpression::try_from_value(value)?; - let fill = NuExpression::try_from_value(fill)?.into_polars(); - let expr: NuExpression = expr.into_polars().fill_null(fill).into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } else { - let lazy = NuLazyFrame::try_from_value(value)?; - let expr = NuExpression::try_from_value(fill)?.into_polars(); - let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().fill_null(expr)); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } - } -} - -#[cfg(test)] -mod test { - use super::super::super::series::Shift; - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyFillNull {}), Box::new(Shift {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/filter.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/filter.rs deleted file mode 100644 index 5635a77e88..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/filter.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyFilter; - -impl Command for LazyFilter { - fn name(&self) -> &str { - "dfr filter" - } - - fn usage(&self) -> &str { - "Filter dataframe based in expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "filter expression", - SyntaxShape::Any, - "Expression that define the column selection", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Filter dataframe using an expression", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr filter ((dfr col a) >= 4)", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(6), Value::test_int(4)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(2)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value: Value = call.req(engine_state, stack, 0)?; - let expression = NuExpression::try_from_value(value)?; - - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let lazy = NuLazyFrame::new( - lazy.from_eager, - lazy.into_polars().filter(expression.into_polars()), - ); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyFilter {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/flatten.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/flatten.rs deleted file mode 100644 index 602dcbcee3..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/flatten.rs +++ /dev/null @@ -1,126 +0,0 @@ -use super::explode::explode; -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyFlatten; - -impl Command for LazyFlatten { - fn name(&self) -> &str { - "dfr flatten" - } - - fn usage(&self) -> &str { - "An alias for dfr explode." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest( - "columns", - SyntaxShape::String, - "columns to flatten, only applicable for dataframes", - ) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ -Example { - description: "Flatten the specified dataframe", - example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr flatten hobbies | dfr collect", - result: Some( - NuDataFrame::try_from_columns(vec![ - Column::new( - "id".to_string(), - vec![ - Value::test_int(1), - Value::test_int(1), - Value::test_int(2), - Value::test_int(2), - ]), - Column::new( - "name".to_string(), - vec![ - Value::test_string("Mercy"), - Value::test_string("Mercy"), - Value::test_string("Bob"), - Value::test_string("Bob"), - ]), - Column::new( - "hobbies".to_string(), - vec![ - Value::test_string("Cycling"), - Value::test_string("Knitting"), - Value::test_string("Skiing"), - Value::test_string("Football"), - ]), - ], None).expect("simple df for test should not fail") - .into_value(Span::test_data()), - ) - }, - Example { - description: "Select a column and flatten the values", - example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr select (dfr col hobbies | dfr flatten)", - result: Some( - NuDataFrame::try_from_columns(vec![ - Column::new( - "hobbies".to_string(), - vec![ - Value::test_string("Cycling"), - Value::test_string("Knitting"), - Value::test_string("Skiing"), - Value::test_string("Football"), - ]), - ], None).expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - explode(call, input) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example}; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(LazyFlatten {})]); - test_dataframe_example(&mut engine_state, &LazyFlatten.examples()[0]); - } - - #[ignore] - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(LazyFlatten {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &LazyFlatten.examples()[1]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs deleted file mode 100644 index c31d563eb6..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs +++ /dev/null @@ -1,161 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame, NuLazyGroupBy}; -use nu_engine::command_prelude::*; - -use polars::prelude::Expr; - -#[derive(Clone)] -pub struct ToLazyGroupBy; - -impl Command for ToLazyGroupBy { - fn name(&self) -> &str { - "dfr group-by" - } - - fn usage(&self) -> &str { - "Creates a group-by object that can be used for other aggregations." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest( - "Group-by expressions", - SyntaxShape::Any, - "Expression(s) that define the lazy group-by", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Group by and perform an aggregation", - example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]] - | dfr into-df - | dfr group-by a - | dfr agg [ - (dfr col b | dfr min | dfr as "b_min") - (dfr col b | dfr max | dfr as "b_max") - (dfr col b | dfr sum | dfr as "b_sum") - ]"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(2)], - ), - Column::new( - "b_min".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - Column::new( - "b_max".to_string(), - vec![Value::test_int(4), Value::test_int(6)], - ), - Column::new( - "b_sum".to_string(), - vec![Value::test_int(6), Value::test_int(10)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Group by and perform an aggregation", - example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]] - | dfr into-lazy - | dfr group-by a - | dfr agg [ - (dfr col b | dfr min | dfr as "b_min") - (dfr col b | dfr max | dfr as "b_max") - (dfr col b | dfr sum | dfr as "b_sum") - ] - | dfr collect"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(2)], - ), - Column::new( - "b_min".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - Column::new( - "b_max".to_string(), - vec![Value::test_int(4), Value::test_int(6)], - ), - Column::new( - "b_sum".to_string(), - vec![Value::test_int(6), Value::test_int(10)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let vals: Vec = call.rest(engine_state, stack, 0)?; - let value = Value::list(vals, call.head); - let expressions = NuExpression::extract_exprs(value)?; - - if expressions - .iter() - .any(|expr| !matches!(expr, Expr::Column(..))) - { - let value: Value = call.req(engine_state, stack, 0)?; - return Err(ShellError::IncompatibleParametersSingle { - msg: "Expected only Col expressions".into(), - span: value.span(), - }); - } - - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let group_by = NuLazyGroupBy { - schema: lazy.schema.clone(), - from_eager: lazy.from_eager, - group_by: Some(lazy.into_polars().group_by(&expressions)), - }; - - Ok(PipelineData::Value(group_by.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::expressions::{ExprAlias, ExprMax, ExprMin, ExprSum}; - use crate::dataframe::lazy::aggregate::LazyAggregate; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - Box::new(ExprAlias {}), - Box::new(ExprMin {}), - Box::new(ExprMax {}), - Box::new(ExprSum {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs deleted file mode 100644 index 4ae297acfd..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs +++ /dev/null @@ -1,252 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{Expr, JoinType}; - -#[derive(Clone)] -pub struct LazyJoin; - -impl Command for LazyJoin { - fn name(&self) -> &str { - "dfr join" - } - - fn usage(&self) -> &str { - "Joins a lazy frame with other lazy frame." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("other", SyntaxShape::Any, "LazyFrame to join with") - .required("left_on", SyntaxShape::Any, "Left column(s) to join on") - .required("right_on", SyntaxShape::Any, "Right column(s) to join on") - .switch( - "inner", - "inner join between lazyframes (default)", - Some('i'), - ) - .switch("left", "left join between lazyframes", Some('l')) - .switch("outer", "outer join between lazyframes", Some('o')) - .switch("cross", "cross join between lazyframes", Some('c')) - .named( - "suffix", - SyntaxShape::String, - "Suffix to use on columns with same name", - Some('s'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Join two lazy dataframes", - example: r#"let df_a = ([[a b c];[1 "a" 0] [2 "b" 1] [1 "c" 2] [1 "c" 3]] | dfr into-lazy); - let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | dfr into-lazy); - $df_a | dfr join $df_b a foo | dfr collect"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![ - Value::test_int(1), - Value::test_int(2), - Value::test_int(1), - Value::test_int(1), - ], - ), - Column::new( - "b".to_string(), - vec![ - Value::test_string("a"), - Value::test_string("b"), - Value::test_string("c"), - Value::test_string("c"), - ], - ), - Column::new( - "c".to_string(), - vec![ - Value::test_int(0), - Value::test_int(1), - Value::test_int(2), - Value::test_int(3), - ], - ), - Column::new( - "bar".to_string(), - vec![ - Value::test_string("a"), - Value::test_string("c"), - Value::test_string("a"), - Value::test_string("a"), - ], - ), - Column::new( - "ham".to_string(), - vec![ - Value::test_string("let"), - Value::test_string("var"), - Value::test_string("let"), - Value::test_string("let"), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Join one eager dataframe with a lazy dataframe", - example: r#"let df_a = ([[a b c];[1 "a" 0] [2 "b" 1] [1 "c" 2] [1 "c" 3]] | dfr into-df); - let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | dfr into-lazy); - $df_a | dfr join $df_b a foo"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![ - Value::test_int(1), - Value::test_int(2), - Value::test_int(1), - Value::test_int(1), - ], - ), - Column::new( - "b".to_string(), - vec![ - Value::test_string("a"), - Value::test_string("b"), - Value::test_string("c"), - Value::test_string("c"), - ], - ), - Column::new( - "c".to_string(), - vec![ - Value::test_int(0), - Value::test_int(1), - Value::test_int(2), - Value::test_int(3), - ], - ), - Column::new( - "bar".to_string(), - vec![ - Value::test_string("a"), - Value::test_string("c"), - Value::test_string("a"), - Value::test_string("a"), - ], - ), - Column::new( - "ham".to_string(), - vec![ - Value::test_string("let"), - Value::test_string("var"), - Value::test_string("let"), - Value::test_string("let"), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let left = call.has_flag(engine_state, stack, "left")?; - let outer = call.has_flag(engine_state, stack, "outer")?; - let cross = call.has_flag(engine_state, stack, "cross")?; - - let how = if left { - JoinType::Left - } else if outer { - JoinType::Outer { coalesce: true } - } else if cross { - JoinType::Cross - } else { - JoinType::Inner - }; - - let other: Value = call.req(engine_state, stack, 0)?; - let other = NuLazyFrame::try_from_value(other)?; - let other = other.into_polars(); - - let left_on: Value = call.req(engine_state, stack, 1)?; - let left_on = NuExpression::extract_exprs(left_on)?; - - let right_on: Value = call.req(engine_state, stack, 2)?; - let right_on = NuExpression::extract_exprs(right_on)?; - - if left_on.len() != right_on.len() { - let right_on: Value = call.req(engine_state, stack, 2)?; - return Err(ShellError::IncompatibleParametersSingle { - msg: "The right column list has a different size to the left column list".into(), - span: right_on.span(), - }); - } - - // Checking that both list of expressions are made out of col expressions or strings - for (index, list) in &[(1usize, &left_on), (2, &left_on)] { - if list.iter().any(|expr| !matches!(expr, Expr::Column(..))) { - let value: Value = call.req(engine_state, stack, *index)?; - return Err(ShellError::IncompatibleParametersSingle { - msg: "Expected only a string, col expressions or list of strings".into(), - span: value.span(), - }); - } - } - - let suffix: Option = call.get_flag(engine_state, stack, "suffix")?; - let suffix = suffix.unwrap_or_else(|| "_x".into()); - - let value = input.into_value(call.head)?; - let lazy = NuLazyFrame::try_from_value(value)?; - let from_eager = lazy.from_eager; - let lazy = lazy.into_polars(); - - let lazy = lazy - .join_builder() - .with(other) - .left_on(left_on) - .right_on(right_on) - .how(how) - .force_parallel(true) - .suffix(suffix) - .finish(); - - let lazy = NuLazyFrame::new(from_eager, lazy); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyJoin {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/macro_commands.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/macro_commands.rs deleted file mode 100644 index 89655b8e3f..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/macro_commands.rs +++ /dev/null @@ -1,246 +0,0 @@ -/// Definition of multiple lazyframe commands using a macro rule -/// All of these commands have an identical body and only require -/// to have a change in the name, description and function -use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame}; -use nu_engine::command_prelude::*; - -macro_rules! lazy_command { - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().$func()); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new($command {})]) - } - } - }; - - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident, $ddot: expr) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().$func($ddot)); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new($command {})]) - } - } - }; - - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident?, $test: ident) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - - let lazy = NuLazyFrame::new( - lazy.from_eager, - lazy.into_polars() - .$func() - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - help: None, - span: None, - inner: vec![], - })?, - ); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new($command {})]) - } - } - }; -} - -// LazyReverse command -// Expands to a command definition for reverse -lazy_command!( - LazyReverse, - "dfr reverse", - "Reverses the LazyFrame", - vec![Example { - description: "Reverses the dataframe.", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr reverse", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(2), Value::test_int(4), Value::test_int(6),], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(2), Value::test_int(2),], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - },], - reverse, - test_reverse -); - -// LazyCache command -// Expands to a command definition for cache -lazy_command!( - LazyCache, - "dfr cache", - "Caches operations in a new LazyFrame.", - vec![Example { - description: "Caches the result into a new LazyFrame", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr reverse | dfr cache", - result: None, - }], - cache, - test_cache -); - -// LazyMedian command -// Expands to a command definition for median aggregation -lazy_command!( - LazyMedian, - "dfr median", - "Aggregates columns to their median value", - vec![Example { - description: "Median value from columns in a dataframe", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr median", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_float(4.0)],), - Column::new("b".to_string(), vec![Value::test_float(2.0)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - },], - median?, - test_median -); diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/mod.rs deleted file mode 100644 index cbbc4e8589..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/mod.rs +++ /dev/null @@ -1,65 +0,0 @@ -pub mod aggregate; -mod collect; -mod explode; -mod fetch; -mod fill_nan; -mod fill_null; -mod filter; -mod flatten; -pub mod groupby; -mod join; -mod macro_commands; -mod quantile; -mod select; -mod sort_by_expr; -mod to_lazy; - -use nu_protocol::engine::StateWorkingSet; - -use crate::dataframe::lazy::aggregate::LazyAggregate; -pub use crate::dataframe::lazy::collect::LazyCollect; -use crate::dataframe::lazy::fetch::LazyFetch; -use crate::dataframe::lazy::fill_nan::LazyFillNA; -pub use crate::dataframe::lazy::fill_null::LazyFillNull; -use crate::dataframe::lazy::filter::LazyFilter; -use crate::dataframe::lazy::groupby::ToLazyGroupBy; -use crate::dataframe::lazy::join::LazyJoin; -pub(crate) use crate::dataframe::lazy::macro_commands::*; -use crate::dataframe::lazy::quantile::LazyQuantile; -pub(crate) use crate::dataframe::lazy::select::LazySelect; -use crate::dataframe::lazy::sort_by_expr::LazySortBy; -pub use crate::dataframe::lazy::to_lazy::ToLazyFrame; -pub use explode::LazyExplode; -pub use flatten::LazyFlatten; - -pub fn add_lazy_decls(working_set: &mut StateWorkingSet) { - macro_rules! bind_command { - ( $command:expr ) => { - working_set.add_decl(Box::new($command)); - }; - ( $( $command:expr ),* ) => { - $( working_set.add_decl(Box::new($command)); )* - }; - } - - // Dataframe commands - bind_command!( - LazyAggregate, - LazyCache, - LazyCollect, - LazyFetch, - LazyFillNA, - LazyFillNull, - LazyFilter, - LazyJoin, - LazyQuantile, - LazyMedian, - LazyReverse, - LazySelect, - LazySortBy, - ToLazyFrame, - ToLazyGroupBy, - LazyExplode, - LazyFlatten - ); -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs deleted file mode 100644 index ac8ec590c6..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs +++ /dev/null @@ -1,87 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{lit, QuantileInterpolOptions}; - -#[derive(Clone)] -pub struct LazyQuantile; - -impl Command for LazyQuantile { - fn name(&self) -> &str { - "dfr quantile" - } - - fn usage(&self) -> &str { - "Aggregates the columns to the selected quantile." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "quantile", - SyntaxShape::Number, - "quantile value for quantile operation", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "quantile value from columns in a dataframe", - example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr quantile 0.5", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_float(4.0)]), - Column::new("b".to_string(), vec![Value::test_float(2.0)]), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - let quantile: f64 = call.req(engine_state, stack, 0)?; - - let lazy = NuLazyFrame::try_from_value(value)?; - let lazy = NuLazyFrame::new( - lazy.from_eager, - lazy.into_polars() - .quantile(lit(quantile), QuantileInterpolOptions::default()) - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - help: None, - span: None, - inner: vec![], - })?, - ); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyQuantile {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/select.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/select.rs deleted file mode 100644 index b4f01bdc07..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/select.rs +++ /dev/null @@ -1,75 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazySelect; - -impl Command for LazySelect { - fn name(&self) -> &str { - "dfr select" - } - - fn usage(&self) -> &str { - "Selects columns from lazyframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest( - "select expressions", - SyntaxShape::Any, - "Expression(s) that define the column selection", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Select a column from the dataframe", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr select a", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "a".to_string(), - vec![Value::test_int(6), Value::test_int(4), Value::test_int(2)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let vals: Vec = call.rest(engine_state, stack, 0)?; - let value = Value::list(vals, call.head); - let expressions = NuExpression::extract_exprs(value)?; - - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().select(&expressions)); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazySelect {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/sort_by_expr.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/sort_by_expr.rs deleted file mode 100644 index 2e109338a9..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/sort_by_expr.rs +++ /dev/null @@ -1,159 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; -use polars::chunked_array::ops::SortMultipleOptions; - -#[derive(Clone)] -pub struct LazySortBy; - -impl Command for LazySortBy { - fn name(&self) -> &str { - "dfr sort-by" - } - - fn usage(&self) -> &str { - "Sorts a lazy dataframe based on expression(s)." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest( - "sort expression", - SyntaxShape::Any, - "sort expression for the dataframe", - ) - .named( - "reverse", - SyntaxShape::List(Box::new(SyntaxShape::Boolean)), - "Reverse sorting. Default is false", - Some('r'), - ) - .switch( - "nulls-last", - "nulls are shown last in the dataframe", - Some('n'), - ) - .switch("maintain-order", "Maintains order during sort", Some('m')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Sort dataframe by one column", - example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr sort-by a", - result: Some( - NuDataFrame::try_from_columns(vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(4), Value::test_int(6)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(4), Value::test_int(1), Value::test_int(2)], - ), - ], None) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Sort column using two columns", - example: - "[[a b]; [6 2] [1 1] [1 4] [2 4]] | dfr into-df | dfr sort-by [a b] -r [false true]", - result: Some( - NuDataFrame::try_from_columns(vec![ - Column::new( - "a".to_string(), - vec![ - Value::test_int(1), - Value::test_int(1), - Value::test_int(2), - Value::test_int(6), - ], - ), - Column::new( - "b".to_string(), - vec![ - Value::test_int(4), - Value::test_int(1), - Value::test_int(4), - Value::test_int(2), - ], - ), - ], None) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let vals: Vec = call.rest(engine_state, stack, 0)?; - let value = Value::list(vals, call.head); - let expressions = NuExpression::extract_exprs(value)?; - let nulls_last = call.has_flag(engine_state, stack, "nulls-last")?; - let maintain_order = call.has_flag(engine_state, stack, "maintain-order")?; - - let reverse: Option> = call.get_flag(engine_state, stack, "reverse")?; - let reverse = match reverse { - Some(list) => { - if expressions.len() != list.len() { - let span = call - .get_flag::(engine_state, stack, "reverse")? - .expect("already checked and it exists") - .span(); - return Err(ShellError::GenericError { - error: "Incorrect list size".into(), - msg: "Size doesn't match expression list".into(), - span: Some(span), - help: None, - inner: vec![], - }); - } else { - list - } - } - None => expressions.iter().map(|_| false).collect::>(), - }; - - let sort_options = SortMultipleOptions { - descending: reverse, - nulls_last, - multithreaded: true, - maintain_order, - }; - - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let lazy = NuLazyFrame::new( - lazy.from_eager, - lazy.into_polars().sort_by_exprs(&expressions, sort_options), - ); - - Ok(PipelineData::Value( - NuLazyFrame::into_value(lazy, call.head)?, - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazySortBy {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/to_lazy.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/to_lazy.rs deleted file mode 100644 index 1c711cdd57..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/to_lazy.rs +++ /dev/null @@ -1,53 +0,0 @@ -use crate::dataframe::values::{NuDataFrame, NuLazyFrame, NuSchema}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ToLazyFrame; - -impl Command for ToLazyFrame { - fn name(&self) -> &str { - "dfr into-lazy" - } - - fn usage(&self) -> &str { - "Converts a dataframe into a lazy dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .named( - "schema", - SyntaxShape::Record(vec![]), - r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#, - Some('s'), - ) - .input_output_type(Type::Any, Type::Custom("dataframe".into())) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Takes a dictionary and creates a lazy dataframe", - example: "[[a b];[1 2] [3 4]] | dfr into-lazy", - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let maybe_schema = call - .get_flag(engine_state, stack, "schema")? - .map(|schema| NuSchema::try_from(&schema)) - .transpose()?; - - let df = NuDataFrame::try_from_iter(input.into_iter(), maybe_schema)?; - let lazy = NuLazyFrame::from_dataframe(df); - let value = Value::custom(Box::new(lazy), call.head); - Ok(PipelineData::Value(value, None)) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/mod.rs deleted file mode 100644 index d99ce516be..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/mod.rs +++ /dev/null @@ -1,36 +0,0 @@ -mod eager; -mod expressions; -mod lazy; -mod series; -mod stub; -mod utils; -mod values; - -pub use eager::add_eager_decls; -pub use expressions::add_expressions; -pub use lazy::add_lazy_decls; -pub use series::add_series_decls; - -use nu_protocol::engine::{EngineState, StateWorkingSet}; - -pub fn add_dataframe_context(mut engine_state: EngineState) -> EngineState { - let delta = { - let mut working_set = StateWorkingSet::new(&engine_state); - working_set.add_decl(Box::new(stub::Dfr)); - add_series_decls(&mut working_set); - add_eager_decls(&mut working_set); - add_expressions(&mut working_set); - add_lazy_decls(&mut working_set); - - working_set.render() - }; - - if let Err(err) = engine_state.merge_delta(delta) { - eprintln!("Error creating dataframe command context: {err:?}"); - } - - engine_state -} - -#[cfg(test)] -mod test_dataframe; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/all_false.rs b/crates/nu-cmd-dataframe/src/dataframe/series/all_false.rs deleted file mode 100644 index 66921e793c..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/all_false.rs +++ /dev/null @@ -1,108 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct AllFalse; - -impl Command for AllFalse { - fn name(&self) -> &str { - "dfr all-false" - } - - fn usage(&self) -> &str { - "Returns true if all values are false." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Returns true if all values are false", - example: "[false false false] | dfr into-df | dfr all-false", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "all_false".to_string(), - vec![Value::test_bool(true)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Checks the result from a comparison", - example: r#"let s = ([5 6 2 10] | dfr into-df); - let res = ($s > 9); - $res | dfr all-false"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "all_false".to_string(), - vec![Value::test_bool(false)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let series = df.as_series(call.head)?; - let bool = series.bool().map_err(|_| ShellError::GenericError { - error: "Error converting to bool".into(), - msg: "all-false only works with series of type bool".into(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let value = Value::bool(!bool.any(), call.head); - - NuDataFrame::try_from_columns( - vec![Column::new("all_false".to_string(), vec![value])], - None, - ) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(AllFalse {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/all_true.rs b/crates/nu-cmd-dataframe/src/dataframe/series/all_true.rs deleted file mode 100644 index 16b4a9edd9..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/all_true.rs +++ /dev/null @@ -1,105 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct AllTrue; - -impl Command for AllTrue { - fn name(&self) -> &str { - "dfr all-true" - } - - fn usage(&self) -> &str { - "Returns true if all values are true." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Returns true if all values are true", - example: "[true true true] | dfr into-df | dfr all-true", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "all_true".to_string(), - vec![Value::test_bool(true)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Checks the result from a comparison", - example: r#"let s = ([5 6 2 8] | dfr into-df); - let res = ($s > 9); - $res | dfr all-true"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "all_true".to_string(), - vec![Value::test_bool(false)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let series = df.as_series(call.head)?; - let bool = series.bool().map_err(|_| ShellError::GenericError { - error: "Error converting to bool".into(), - msg: "all-false only works with series of type bool".into(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let value = Value::bool(bool.all(), call.head); - - NuDataFrame::try_from_columns(vec![Column::new("all_true".to_string(), vec![value])], None) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(AllTrue {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/arg_max.rs b/crates/nu-cmd-dataframe/src/dataframe/series/arg_max.rs deleted file mode 100644 index d7539401ab..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/arg_max.rs +++ /dev/null @@ -1,85 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{ArgAgg, IntoSeries, NewChunkedArray, UInt32Chunked}; - -#[derive(Clone)] -pub struct ArgMax; - -impl Command for ArgMax { - fn name(&self) -> &str { - "dfr arg-max" - } - - fn usage(&self) -> &str { - "Return index for max value in series." - } - - fn search_terms(&self) -> Vec<&str> { - vec!["argmax", "maximum", "most", "largest", "greatest"] - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns index for max value", - example: "[1 3 2] | dfr into-df | dfr arg-max", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new("arg_max".to_string(), vec![Value::test_int(1)])], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let res = series.arg_max(); - let chunked = match res { - Some(index) => UInt32Chunked::from_slice("arg_max", &[index as u32]), - None => UInt32Chunked::from_slice("arg_max", &[]), - }; - - let res = chunked.into_series(); - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ArgMax {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/arg_min.rs b/crates/nu-cmd-dataframe/src/dataframe/series/arg_min.rs deleted file mode 100644 index 1b685d65b4..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/arg_min.rs +++ /dev/null @@ -1,85 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{ArgAgg, IntoSeries, NewChunkedArray, UInt32Chunked}; - -#[derive(Clone)] -pub struct ArgMin; - -impl Command for ArgMin { - fn name(&self) -> &str { - "dfr arg-min" - } - - fn usage(&self) -> &str { - "Return index for min value in series." - } - - fn search_terms(&self) -> Vec<&str> { - vec!["argmin", "minimum", "least", "smallest", "lowest"] - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns index for min value", - example: "[1 3 2] | dfr into-df | dfr arg-min", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new("arg_min".to_string(), vec![Value::test_int(0)])], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let res = series.arg_min(); - let chunked = match res { - Some(index) => UInt32Chunked::from_slice("arg_min", &[index as u32]), - None => UInt32Chunked::from_slice("arg_min", &[]), - }; - - let res = chunked.into_series(); - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ArgMin {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/cumulative.rs b/crates/nu-cmd-dataframe/src/dataframe/series/cumulative.rs deleted file mode 100644 index c32875e87b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/cumulative.rs +++ /dev/null @@ -1,148 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{DataType, IntoSeries}; -use polars_ops::prelude::{cum_max, cum_min, cum_sum}; - -enum CumType { - Min, - Max, - Sum, -} - -impl CumType { - fn from_str(roll_type: &str, span: Span) -> Result { - match roll_type { - "min" => Ok(Self::Min), - "max" => Ok(Self::Max), - "sum" => Ok(Self::Sum), - _ => Err(ShellError::GenericError { - error: "Wrong operation".into(), - msg: "Operation not valid for cumulative".into(), - span: Some(span), - help: Some("Allowed values: max, min, sum".into()), - inner: vec![], - }), - } - } - - fn to_str(&self) -> &'static str { - match self { - CumType::Min => "cumulative_min", - CumType::Max => "cumulative_max", - CumType::Sum => "cumulative_sum", - } - } -} - -#[derive(Clone)] -pub struct Cumulative; - -impl Command for Cumulative { - fn name(&self) -> &str { - "dfr cumulative" - } - - fn usage(&self) -> &str { - "Cumulative calculation for a series." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("type", SyntaxShape::String, "rolling operation") - .switch("reverse", "Reverse cumulative calculation", Some('r')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Cumulative sum for a series", - example: "[1 2 3 4 5] | dfr into-df | dfr cumulative sum", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0_cumulative_sum".to_string(), - vec![ - Value::test_int(1), - Value::test_int(3), - Value::test_int(6), - Value::test_int(10), - Value::test_int(15), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let cum_type: Spanned = call.req(engine_state, stack, 0)?; - let reverse = call.has_flag(engine_state, stack, "reverse")?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - if let DataType::Object(..) = series.dtype() { - return Err(ShellError::GenericError { - error: "Found object series".into(), - msg: "Series of type object cannot be used for cumulative operation".into(), - span: Some(call.head), - help: None, - inner: vec![], - }); - } - - let cum_type = CumType::from_str(&cum_type.item, cum_type.span)?; - let mut res = match cum_type { - CumType::Max => cum_max(&series, reverse), - CumType::Min => cum_min(&series, reverse), - CumType::Sum => cum_sum(&series, reverse), - } - .map_err(|e| ShellError::GenericError { - error: "Error creating cumulative".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let name = format!("{}_{}", series.name(), cum_type.to_str()); - res.rename(&name); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Cumulative {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/as_date.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/as_date.rs deleted file mode 100644 index b406057572..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/as_date.rs +++ /dev/null @@ -1,94 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, StringMethods}; - -#[derive(Clone)] -pub struct AsDate; - -impl Command for AsDate { - fn name(&self) -> &str { - "dfr as-date" - } - - fn usage(&self) -> &str { - r#"Converts string to date."# - } - - fn extra_usage(&self) -> &str { - r#"Format example: - "%Y-%m-%d" => 2021-12-31 - "%d-%m-%Y" => 31-12-2021 - "%Y%m%d" => 2021319 (2021-03-19)"# - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("format", SyntaxShape::String, "formatting date string") - .switch("not-exact", "the format string may be contained in the date (e.g. foo-2021-01-01-bar could match 2021-01-01)", Some('n')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Converts string to date", - example: r#"["2021-12-30" "2021-12-31"] | dfr into-df | dfr as-datetime "%Y-%m-%d""#, - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let format: String = call.req(engine_state, stack, 0)?; - let not_exact = call.has_flag(engine_state, stack, "not-exact")?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - let casted = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = if not_exact { - casted.as_date_not_exact(Some(format.as_str())) - } else { - casted.as_date(Some(format.as_str()), false) - }; - - let mut res = res - .map_err(|e| ShellError::GenericError { - error: "Error creating datetime".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - - res.rename("date"); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs deleted file mode 100644 index 6ee979b069..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs +++ /dev/null @@ -1,187 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use chrono::DateTime; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, StringMethods, TimeUnit}; - -#[derive(Clone)] -pub struct AsDateTime; - -impl Command for AsDateTime { - fn name(&self) -> &str { - "dfr as-datetime" - } - - fn usage(&self) -> &str { - r#"Converts string to datetime."# - } - - fn extra_usage(&self) -> &str { - r#"Format example: - "%y/%m/%d %H:%M:%S" => 21/12/31 12:54:98 - "%y-%m-%d %H:%M:%S" => 2021-12-31 24:58:01 - "%y/%m/%d %H:%M:%S" => 21/12/31 24:58:01 - "%y%m%d %H:%M:%S" => 210319 23:58:50 - "%Y/%m/%d %H:%M:%S" => 2021/12/31 12:54:98 - "%Y-%m-%d %H:%M:%S" => 2021-12-31 24:58:01 - "%Y/%m/%d %H:%M:%S" => 2021/12/31 24:58:01 - "%Y%m%d %H:%M:%S" => 20210319 23:58:50 - "%FT%H:%M:%S" => 2019-04-18T02:45:55 - "%FT%H:%M:%S.%6f" => microseconds - "%FT%H:%M:%S.%9f" => nanoseconds"# - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("format", SyntaxShape::String, "formatting date time string") - .switch("not-exact", "the format string may be contained in the date (e.g. foo-2021-01-01-bar could match 2021-01-01)", Some('n')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Converts string to datetime", - example: r#"["2021-12-30 00:00:00" "2021-12-31 00:00:00"] | dfr into-df | dfr as-datetime "%Y-%m-%d %H:%M:%S""#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "datetime".to_string(), - vec![ - Value::date( - DateTime::parse_from_str( - "2021-12-30 00:00:00 +0000", - "%Y-%m-%d %H:%M:%S %z", - ) - .expect("date calculation should not fail in test"), - Span::test_data(), - ), - Value::date( - DateTime::parse_from_str( - "2021-12-31 00:00:00 +0000", - "%Y-%m-%d %H:%M:%S %z", - ) - .expect("date calculation should not fail in test"), - Span::test_data(), - ), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Converts string to datetime with high resolutions", - example: r#"["2021-12-30 00:00:00.123456789" "2021-12-31 00:00:00.123456789"] | dfr into-df | dfr as-datetime "%Y-%m-%d %H:%M:%S.%9f""#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "datetime".to_string(), - vec![ - Value::date( - DateTime::parse_from_str( - "2021-12-30 00:00:00.123456789 +0000", - "%Y-%m-%d %H:%M:%S.%9f %z", - ) - .expect("date calculation should not fail in test"), - Span::test_data(), - ), - Value::date( - DateTime::parse_from_str( - "2021-12-31 00:00:00.123456789 +0000", - "%Y-%m-%d %H:%M:%S.%9f %z", - ) - .expect("date calculation should not fail in test"), - Span::test_data(), - ), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let format: String = call.req(engine_state, stack, 0)?; - let not_exact = call.has_flag(engine_state, stack, "not-exact")?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - let casted = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = if not_exact { - casted.as_datetime_not_exact( - Some(format.as_str()), - TimeUnit::Nanoseconds, - false, - None, - &Default::default(), - ) - } else { - casted.as_datetime( - Some(format.as_str()), - TimeUnit::Nanoseconds, - false, - false, - None, - &Default::default(), - ) - }; - - let mut res = res - .map_err(|e| ShellError::GenericError { - error: "Error creating datetime".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - - res.rename("datetime"); - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(AsDateTime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_day.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_day.rs deleted file mode 100644 index 9187219d7a..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_day.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetDay; - -impl Command for GetDay { - fn name(&self) -> &str { - "dfr get-day" - } - - fn usage(&self) -> &str { - "Gets day from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns day from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-day"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(4), Value::test_int(4)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.day().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetDay {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_hour.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_hour.rs deleted file mode 100644 index ba05843047..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_hour.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetHour; - -impl Command for GetHour { - fn name(&self) -> &str { - "dfr get-hour" - } - - fn usage(&self) -> &str { - "Gets hour from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns hour from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-hour"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(16), Value::test_int(16)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.hour().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetHour {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_minute.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_minute.rs deleted file mode 100644 index 902ed61d56..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_minute.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetMinute; - -impl Command for GetMinute { - fn name(&self) -> &str { - "dfr get-minute" - } - - fn usage(&self) -> &str { - "Gets minute from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns minute from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-minute"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(39), Value::test_int(39)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.minute().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetMinute {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_month.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_month.rs deleted file mode 100644 index 077d5afc1e..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_month.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetMonth; - -impl Command for GetMonth { - fn name(&self) -> &str { - "dfr get-month" - } - - fn usage(&self) -> &str { - "Gets month from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns month from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-month"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(8), Value::test_int(8)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.month().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetMonth {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_nanosecond.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_nanosecond.rs deleted file mode 100644 index 1543e31082..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_nanosecond.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetNanosecond; - -impl Command for GetNanosecond { - fn name(&self) -> &str { - "dfr get-nanosecond" - } - - fn usage(&self) -> &str { - "Gets nanosecond from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns nanosecond from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-nanosecond"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(0), Value::test_int(0)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.nanosecond().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetNanosecond {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_ordinal.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_ordinal.rs deleted file mode 100644 index b77ebbc14c..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_ordinal.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetOrdinal; - -impl Command for GetOrdinal { - fn name(&self) -> &str { - "dfr get-ordinal" - } - - fn usage(&self) -> &str { - "Gets ordinal from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns ordinal from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-ordinal"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(217), Value::test_int(217)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.ordinal().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetOrdinal {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_second.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_second.rs deleted file mode 100644 index e039bcc010..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_second.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetSecond; - -impl Command for GetSecond { - fn name(&self) -> &str { - "dfr get-second" - } - - fn usage(&self) -> &str { - "Gets second from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns second from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-second"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(18), Value::test_int(18)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.second().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetSecond {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_week.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_week.rs deleted file mode 100644 index 1a1bc2c12d..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_week.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetWeek; - -impl Command for GetWeek { - fn name(&self) -> &str { - "dfr get-week" - } - - fn usage(&self) -> &str { - "Gets week from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns week from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-week"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(32), Value::test_int(32)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.week().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetWeek {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_weekday.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_weekday.rs deleted file mode 100644 index b5cf1b3197..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_weekday.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetWeekDay; - -impl Command for GetWeekDay { - fn name(&self) -> &str { - "dfr get-weekday" - } - - fn usage(&self) -> &str { - "Gets weekday from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns weekday from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-weekday"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(2), Value::test_int(2)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.weekday().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetWeekDay {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_year.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_year.rs deleted file mode 100644 index 1ec3515949..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_year.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetYear; - -impl Command for GetYear { - fn name(&self) -> &str { - "dfr get-year" - } - - fn usage(&self) -> &str { - "Gets year from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns year from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-year"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(2020), Value::test_int(2020)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.year().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetYear {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/mod.rs deleted file mode 100644 index ed3895a172..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -mod as_date; -mod as_datetime; -mod get_day; -mod get_hour; -mod get_minute; -mod get_month; -mod get_nanosecond; -mod get_ordinal; -mod get_second; -mod get_week; -mod get_weekday; -mod get_year; - -pub use as_date::AsDate; -pub use as_datetime::AsDateTime; -pub use get_day::GetDay; -pub use get_hour::GetHour; -pub use get_minute::GetMinute; -pub use get_month::GetMonth; -pub use get_nanosecond::GetNanosecond; -pub use get_ordinal::GetOrdinal; -pub use get_second::GetSecond; -pub use get_week::GetWeek; -pub use get_weekday::GetWeekDay; -pub use get_year::GetYear; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_sort.rs b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_sort.rs deleted file mode 100644 index bf28cbac58..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_sort.rs +++ /dev/null @@ -1,130 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, SortOptions}; - -#[derive(Clone)] -pub struct ArgSort; - -impl Command for ArgSort { - fn name(&self) -> &str { - "dfr arg-sort" - } - - fn usage(&self) -> &str { - "Returns indexes for a sorted series." - } - - fn search_terms(&self) -> Vec<&str> { - vec!["argsort", "order", "arrange"] - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .switch("reverse", "reverse order", Some('r')) - .switch("nulls-last", "nulls ordered last", Some('n')) - .switch( - "maintain-order", - "maintain order on sorted items", - Some('m'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Returns indexes for a sorted series", - example: "[1 2 2 3 3] | dfr into-df | dfr arg-sort", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "arg_sort".to_string(), - vec![ - Value::test_int(0), - Value::test_int(1), - Value::test_int(2), - Value::test_int(3), - Value::test_int(4), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Returns indexes for a sorted series", - example: "[1 2 2 3 3] | dfr into-df | dfr arg-sort --reverse", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "arg_sort".to_string(), - vec![ - Value::test_int(3), - Value::test_int(4), - Value::test_int(1), - Value::test_int(2), - Value::test_int(0), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let sort_options = SortOptions { - descending: call.has_flag(engine_state, stack, "reverse")?, - nulls_last: call.has_flag(engine_state, stack, "nulls-last")?, - multithreaded: true, - maintain_order: call.has_flag(engine_state, stack, "maintain-order")?, - }; - - let mut res = df - .as_series(call.head)? - .arg_sort(sort_options) - .into_series(); - res.rename("arg_sort"); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ArgSort {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_true.rs b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_true.rs deleted file mode 100644 index 106e95f5ea..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_true.rs +++ /dev/null @@ -1,115 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{arg_where, col, IntoLazy}; - -#[derive(Clone)] -pub struct ArgTrue; - -impl Command for ArgTrue { - fn name(&self) -> &str { - "dfr arg-true" - } - - fn usage(&self) -> &str { - "Returns indexes where values are true." - } - - fn search_terms(&self) -> Vec<&str> { - vec!["argtrue", "truth", "boolean-true"] - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns indexes where values are true", - example: "[false true false] | dfr into-df | dfr arg-true", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "arg_true".to_string(), - vec![Value::test_int(1)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let columns = df.as_ref().get_column_names(); - if columns.len() > 1 { - return Err(ShellError::GenericError { - error: "Error using as series".into(), - msg: "dataframe has more than one column".into(), - span: Some(call.head), - help: None, - inner: vec![], - }); - } - - match columns.first() { - Some(column) => { - let expression = arg_where(col(column).eq(true)).alias("arg_true"); - let res = df - .as_ref() - .clone() - .lazy() - .select(&[expression]) - .collect() - .map_err(|err| ShellError::GenericError { - error: "Error creating index column".into(), - msg: err.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let value = NuDataFrame::dataframe_into_value(res, call.head); - Ok(PipelineData::Value(value, None)) - } - _ => Err(ShellError::UnsupportedInput { - msg: "Expected the dataframe to have a column".to_string(), - input: "".to_string(), - msg_span: call.head, - input_span: call.head, - }), - } -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ArgTrue {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_unique.rs b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_unique.rs deleted file mode 100644 index 6b69518cba..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_unique.rs +++ /dev/null @@ -1,93 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::IntoSeries; - -#[derive(Clone)] -pub struct ArgUnique; - -impl Command for ArgUnique { - fn name(&self) -> &str { - "dfr arg-unique" - } - - fn usage(&self) -> &str { - "Returns indexes for unique values." - } - - fn search_terms(&self) -> Vec<&str> { - vec!["argunique", "distinct", "noduplicate", "unrepeated"] - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns indexes for unique values", - example: "[1 2 2 3 3] | dfr into-df | dfr arg-unique", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "arg_unique".to_string(), - vec![Value::test_int(0), Value::test_int(1), Value::test_int(3)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut res = df - .as_series(call.head)? - .arg_unique() - .map_err(|e| ShellError::GenericError { - error: "Error extracting unique values".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - res.rename("arg_unique"); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ArgUnique {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/mod.rs deleted file mode 100644 index c0af8c8653..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -mod arg_sort; -mod arg_true; -mod arg_unique; -mod set_with_idx; - -pub use arg_sort::ArgSort; -pub use arg_true::ArgTrue; -pub use arg_unique::ArgUnique; -pub use set_with_idx::SetWithIndex; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/set_with_idx.rs b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/set_with_idx.rs deleted file mode 100644 index 307ef4d5c3..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/set_with_idx.rs +++ /dev/null @@ -1,213 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{ChunkSet, DataType, IntoSeries}; - -#[derive(Clone)] -pub struct SetWithIndex; - -impl Command for SetWithIndex { - fn name(&self) -> &str { - "dfr set-with-idx" - } - - fn usage(&self) -> &str { - "Sets value in the given index." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("value", SyntaxShape::Any, "value to be inserted in series") - .required_named( - "indices", - SyntaxShape::Any, - "list of indices indicating where to set the value", - Some('i'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Set value in selected rows from series", - example: r#"let series = ([4 1 5 2 4 3] | dfr into-df); - let indices = ([0 2] | dfr into-df); - $series | dfr set-with-idx 6 --indices $indices"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_int(6), - Value::test_int(1), - Value::test_int(6), - Value::test_int(2), - Value::test_int(4), - Value::test_int(3), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let value: Value = call.req(engine_state, stack, 0)?; - - let indices_value: Value = call - .get_flag(engine_state, stack, "indices")? - .expect("required named value"); - let indices_span = indices_value.span(); - let indices = NuDataFrame::try_from_value(indices_value)?.as_series(indices_span)?; - - let casted = match indices.dtype() { - DataType::UInt32 | DataType::UInt64 | DataType::Int32 | DataType::Int64 => indices - .as_ref() - .cast(&DataType::UInt32) - .map_err(|e| ShellError::GenericError { - error: "Error casting indices".into(), - msg: e.to_string(), - span: Some(indices_span), - help: None, - inner: vec![], - }), - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: "Series with incorrect type".into(), - span: Some(indices_span), - help: Some("Consider using a Series with type int type".into()), - inner: vec![], - }), - }?; - - let indices = casted - .u32() - .map_err(|e| ShellError::GenericError { - error: "Error casting indices".into(), - msg: e.to_string(), - span: Some(indices_span), - help: None, - inner: vec![], - })? - .into_iter() - .flatten(); - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let span = value.span(); - let res = match value { - Value::Int { val, .. } => { - let chunked = series.i64().map_err(|e| ShellError::GenericError { - error: "Error casting to i64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let res = chunked.scatter_single(indices, Some(val)).map_err(|e| { - ShellError::GenericError { - error: "Error setting value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - } - })?; - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - } - Value::Float { val, .. } => { - let chunked = series.f64().map_err(|e| ShellError::GenericError { - error: "Error casting to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let res = chunked.scatter_single(indices, Some(val)).map_err(|e| { - ShellError::GenericError { - error: "Error setting value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - } - })?; - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - } - Value::String { val, .. } => { - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let res = chunked - .scatter_single(indices, Some(val.as_ref())) - .map_err(|e| ShellError::GenericError { - error: "Error setting value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let mut res = res.into_series(); - res.rename("string"); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - } - _ => Err(ShellError::GenericError { - error: "Incorrect value type".into(), - msg: format!( - "this value cannot be set in a series of type '{}'", - series.dtype() - ), - span: Some(span), - help: None, - inner: vec![], - }), - }; - - res.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(SetWithIndex {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_duplicated.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_duplicated.rs deleted file mode 100644 index b28f977b47..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_duplicated.rs +++ /dev/null @@ -1,122 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::IntoSeries; - -#[derive(Clone)] -pub struct IsDuplicated; - -impl Command for IsDuplicated { - fn name(&self) -> &str { - "dfr is-duplicated" - } - - fn usage(&self) -> &str { - "Creates mask indicating duplicated values." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create mask indicating duplicated values", - example: "[5 6 6 6 8 8 8] | dfr into-df | dfr is-duplicated", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_duplicated".to_string(), - vec![ - Value::test_bool(false), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Create mask indicating duplicated rows in a dataframe", - example: - "[[a, b]; [1 2] [1 2] [3 3] [3 3] [1 1]] | dfr into-df | dfr is-duplicated", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_duplicated".to_string(), - vec![ - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut res = df - .as_ref() - .is_duplicated() - .map_err(|e| ShellError::GenericError { - error: "Error finding duplicates".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - - res.rename("is_duplicated"); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(IsDuplicated {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_in.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_in.rs deleted file mode 100644 index 0792d3fddf..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_in.rs +++ /dev/null @@ -1,104 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{is_in, IntoSeries}; - -#[derive(Clone)] -pub struct IsIn; - -impl Command for IsIn { - fn name(&self) -> &str { - "dfr is-in" - } - - fn usage(&self) -> &str { - "Checks if elements from a series are contained in right series." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("other", SyntaxShape::Any, "right series") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Checks if elements from a series are contained in right series", - example: r#"let other = ([1 3 6] | dfr into-df); - [5 6 6 6 8 8 8] | dfr into-df | dfr is-in $other"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_in".to_string(), - vec![ - Value::test_bool(false), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?.as_series(call.head)?; - - let other_value: Value = call.req(engine_state, stack, 0)?; - let other_span = other_value.span(); - let other_df = NuDataFrame::try_from_value(other_value)?; - let other = other_df.as_series(other_span)?; - - let mut res = is_in(&df, &other) - .map_err(|e| ShellError::GenericError { - error: "Error finding in other".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - - res.rename("is_in"); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(IsIn {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs deleted file mode 100644 index 4ed33ce951..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs +++ /dev/null @@ -1,122 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; -use polars::prelude::IntoSeries; - -#[derive(Clone)] -pub struct IsNotNull; - -impl Command for IsNotNull { - fn name(&self) -> &str { - "dfr is-not-null" - } - - fn usage(&self) -> &str { - "Creates mask where value is not null." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create mask where values are not null", - example: r#"let s = ([5 6 0 8] | dfr into-df); - let res = ($s / $s); - $res | dfr is-not-null"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_not_null".to_string(), - vec![ - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(false), - Value::test_bool(true), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates a is not null expression from a column", - example: "dfr col a | dfr is-not-null", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let df = NuDataFrame::try_from_value(value)?; - command(engine_state, stack, call, df) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().is_not_null().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let mut res = df.as_series(call.head)?.is_not_null(); - res.rename("is_not_null"); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - use crate::dataframe::test_dataframe::{build_test_engine_state, test_dataframe_example}; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(IsNotNull {})]); - test_dataframe_example(&mut engine_state, &IsNotNull.examples()[0]); - } - - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(IsNotNull {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &IsNotNull.examples()[1]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs deleted file mode 100644 index b99d48af66..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs +++ /dev/null @@ -1,122 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; -use polars::prelude::IntoSeries; - -#[derive(Clone)] -pub struct IsNull; - -impl Command for IsNull { - fn name(&self) -> &str { - "dfr is-null" - } - - fn usage(&self) -> &str { - "Creates mask where value is null." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create mask where values are null", - example: r#"let s = ([5 6 0 8] | dfr into-df); - let res = ($s / $s); - $res | dfr is-null"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_null".to_string(), - vec![ - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(true), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates a is null expression from a column", - example: "dfr col a | dfr is-null", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let df = NuDataFrame::try_from_value(value)?; - command(engine_state, stack, call, df) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().is_null().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let mut res = df.as_series(call.head)?.is_null(); - res.rename("is_null"); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - use crate::dataframe::test_dataframe::{build_test_engine_state, test_dataframe_example}; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(IsNull {})]); - test_dataframe_example(&mut engine_state, &IsNull.examples()[0]); - } - - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(IsNull {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &IsNull.examples()[1]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_unique.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_unique.rs deleted file mode 100644 index 8e313abca7..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_unique.rs +++ /dev/null @@ -1,121 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::IntoSeries; - -#[derive(Clone)] -pub struct IsUnique; - -impl Command for IsUnique { - fn name(&self) -> &str { - "dfr is-unique" - } - - fn usage(&self) -> &str { - "Creates mask indicating unique values." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create mask indicating unique values", - example: "[5 6 6 6 8 8 8] | dfr into-df | dfr is-unique", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_unique".to_string(), - vec![ - Value::test_bool(true), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Create mask indicating duplicated rows in a dataframe", - example: "[[a, b]; [1 2] [1 2] [3 3] [3 3] [1 1]] | dfr into-df | dfr is-unique", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_unique".to_string(), - vec![ - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(true), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut res = df - .as_ref() - .is_unique() - .map_err(|e| ShellError::GenericError { - error: "Error finding unique values".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - - res.rename("is_unique"); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(IsUnique {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/mod.rs deleted file mode 100644 index 80c98b5ef0..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -mod is_duplicated; -mod is_in; -mod is_not_null; -mod is_null; -mod is_unique; -mod not; -mod set; - -pub use is_duplicated::IsDuplicated; -pub use is_in::IsIn; -pub use is_not_null::IsNotNull; -pub use is_null::IsNull; -pub use is_unique::IsUnique; -pub use not::NotSeries; -pub use set::SetSeries; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/not.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/not.rs deleted file mode 100644 index 081a3c3b23..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/not.rs +++ /dev/null @@ -1,93 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::IntoSeries; - -use std::ops::Not; - -#[derive(Clone)] -pub struct NotSeries; - -impl Command for NotSeries { - fn name(&self) -> &str { - "dfr not" - } - - fn usage(&self) -> &str { - "Inverts boolean mask." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Inverts boolean mask", - example: "[true false true] | dfr into-df | dfr not", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_bool(false), - Value::test_bool(true), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - command(engine_state, stack, call, df) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let series = df.as_series(call.head)?; - - let bool = series.bool().map_err(|e| ShellError::GenericError { - error: "Error inverting mask".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = bool.not(); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(NotSeries {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/set.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/set.rs deleted file mode 100644 index 4dacb7117b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/set.rs +++ /dev/null @@ -1,201 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{ChunkSet, DataType, IntoSeries}; - -#[derive(Clone)] -pub struct SetSeries; - -impl Command for SetSeries { - fn name(&self) -> &str { - "dfr set" - } - - fn usage(&self) -> &str { - "Sets value where given mask is true." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("value", SyntaxShape::Any, "value to be inserted in series") - .required_named( - "mask", - SyntaxShape::Any, - "mask indicating insertions", - Some('m'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Shifts the values by a given period", - example: r#"let s = ([1 2 2 3 3] | dfr into-df | dfr shift 2); - let mask = ($s | dfr is-null); - $s | dfr set 0 --mask $mask"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_int(0), - Value::test_int(0), - Value::test_int(1), - Value::test_int(2), - Value::test_int(2), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let value: Value = call.req(engine_state, stack, 0)?; - - let mask_value: Value = call - .get_flag(engine_state, stack, "mask")? - .expect("required named value"); - let mask_span = mask_value.span(); - let mask = NuDataFrame::try_from_value(mask_value)?.as_series(mask_span)?; - - let bool_mask = match mask.dtype() { - DataType::Boolean => mask.bool().map_err(|e| ShellError::GenericError { - error: "Error casting to bool".into(), - msg: e.to_string(), - span: Some(mask_span), - help: None, - inner: vec![], - }), - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: "can only use bool series as mask".into(), - span: Some(mask_span), - help: None, - inner: vec![], - }), - }?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - let span = value.span(); - let res = match value { - Value::Int { val, .. } => { - let chunked = series.i64().map_err(|e| ShellError::GenericError { - error: "Error casting to i64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let res = chunked - .set(bool_mask, Some(val)) - .map_err(|e| ShellError::GenericError { - error: "Error setting value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - } - Value::Float { val, .. } => { - let chunked = series.f64().map_err(|e| ShellError::GenericError { - error: "Error casting to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let res = chunked - .set(bool_mask, Some(val)) - .map_err(|e| ShellError::GenericError { - error: "Error setting value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - } - Value::String { val, .. } => { - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let res = chunked.set(bool_mask, Some(val.as_ref())).map_err(|e| { - ShellError::GenericError { - error: "Error setting value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - } - })?; - - let mut res = res.into_series(); - res.rename("string"); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - } - _ => Err(ShellError::GenericError { - error: "Incorrect value type".into(), - msg: format!( - "this value cannot be set in a series of type '{}'", - series.dtype() - ), - span: Some(span), - help: None, - inner: vec![], - }), - }; - - res.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::super::super::{IsNull, Shift}; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(SetSeries {}), - Box::new(IsNull {}), - Box::new(Shift {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/series/mod.rs deleted file mode 100644 index e1b9bc1087..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/mod.rs +++ /dev/null @@ -1,95 +0,0 @@ -mod date; -pub use date::*; - -mod string; -pub use string::*; - -mod masks; -pub use masks::*; - -mod indexes; -pub use indexes::*; - -mod all_false; -mod all_true; -mod arg_max; -mod arg_min; -mod cumulative; -mod n_null; -mod n_unique; -mod rolling; -mod shift; -mod unique; -mod value_counts; - -use nu_protocol::engine::StateWorkingSet; - -pub use all_false::AllFalse; -pub use all_true::AllTrue; -pub use arg_max::ArgMax; -pub use arg_min::ArgMin; -pub use cumulative::Cumulative; -pub use n_null::NNull; -pub use n_unique::NUnique; -pub use rolling::Rolling; -pub use shift::Shift; -pub use unique::Unique; -pub use value_counts::ValueCount; - -pub fn add_series_decls(working_set: &mut StateWorkingSet) { - macro_rules! bind_command { - ( $command:expr ) => { - working_set.add_decl(Box::new($command)); - }; - ( $( $command:expr ),* ) => { - $( working_set.add_decl(Box::new($command)); )* - }; - } - - // Series commands - bind_command!( - AllFalse, - AllTrue, - ArgMax, - ArgMin, - ArgSort, - ArgTrue, - ArgUnique, - AsDate, - AsDateTime, - Concatenate, - Contains, - Cumulative, - GetDay, - GetHour, - GetMinute, - GetMonth, - GetNanosecond, - GetOrdinal, - GetSecond, - GetWeek, - GetWeekDay, - GetYear, - IsDuplicated, - IsIn, - IsNotNull, - IsNull, - IsUnique, - NNull, - NUnique, - NotSeries, - Replace, - ReplaceAll, - Rolling, - SetSeries, - SetWithIndex, - Shift, - StrLengths, - StrSlice, - StrFTime, - ToLowerCase, - ToUpperCase, - Unique, - ValueCount - ); -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/n_null.rs b/crates/nu-cmd-dataframe/src/dataframe/series/n_null.rs deleted file mode 100644 index 6c9909da07..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/n_null.rs +++ /dev/null @@ -1,82 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct NNull; - -impl Command for NNull { - fn name(&self) -> &str { - "dfr count-null" - } - - fn usage(&self) -> &str { - "Counts null values." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Counts null values", - example: r#"let s = ([1 1 0 0 3 3 4] | dfr into-df); - ($s / $s) | dfr count-null"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "count_null".to_string(), - vec![Value::test_int(2)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let res = df.as_series(call.head)?.null_count(); - let value = Value::int(res as i64, call.head); - - NuDataFrame::try_from_columns( - vec![Column::new("count_null".to_string(), vec![value])], - None, - ) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(NNull {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs b/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs deleted file mode 100644 index c6d6e829f8..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs +++ /dev/null @@ -1,127 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct NUnique; - -impl Command for NUnique { - fn name(&self) -> &str { - "dfr n-unique" - } - - fn usage(&self) -> &str { - "Counts unique values." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Counts unique values", - example: "[1 1 2 2 3 3 4] | dfr into-df | dfr n-unique", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "count_unique".to_string(), - vec![Value::test_int(4)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates a is n-unique expression from a column", - example: "dfr col a | dfr n-unique", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let df = NuDataFrame::try_from_value(value)?; - command(engine_state, stack, call, df) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().n_unique().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let res = df - .as_series(call.head)? - .n_unique() - .map_err(|e| ShellError::GenericError { - error: "Error counting unique values".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let value = Value::int(res as i64, call.head); - - NuDataFrame::try_from_columns( - vec![Column::new("count_unique".to_string(), vec![value])], - None, - ) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example}; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(NUnique {})]); - test_dataframe_example(&mut engine_state, &NUnique.examples()[0]); - } - - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(NUnique {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &NUnique.examples()[1]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/rolling.rs b/crates/nu-cmd-dataframe/src/dataframe/series/rolling.rs deleted file mode 100644 index b659462298..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/rolling.rs +++ /dev/null @@ -1,186 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{DataType, Duration, IntoSeries, RollingOptionsImpl, SeriesOpsTime}; - -enum RollType { - Min, - Max, - Sum, - Mean, -} - -impl RollType { - fn from_str(roll_type: &str, span: Span) -> Result { - match roll_type { - "min" => Ok(Self::Min), - "max" => Ok(Self::Max), - "sum" => Ok(Self::Sum), - "mean" => Ok(Self::Mean), - _ => Err(ShellError::GenericError { - error: "Wrong operation".into(), - msg: "Operation not valid for cumulative".into(), - span: Some(span), - help: Some("Allowed values: min, max, sum, mean".into()), - inner: vec![], - }), - } - } - - fn to_str(&self) -> &'static str { - match self { - RollType::Min => "rolling_min", - RollType::Max => "rolling_max", - RollType::Sum => "rolling_sum", - RollType::Mean => "rolling_mean", - } - } -} - -#[derive(Clone)] -pub struct Rolling; - -impl Command for Rolling { - fn name(&self) -> &str { - "dfr rolling" - } - - fn usage(&self) -> &str { - "Rolling calculation for a series." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("type", SyntaxShape::String, "rolling operation") - .required("window", SyntaxShape::Int, "Window size for rolling") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Rolling sum for a series", - example: "[1 2 3 4 5] | dfr into-df | dfr rolling sum 2 | dfr drop-nulls", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0_rolling_sum".to_string(), - vec![ - Value::test_int(3), - Value::test_int(5), - Value::test_int(7), - Value::test_int(9), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Rolling max for a series", - example: "[1 2 3 4 5] | dfr into-df | dfr rolling max 2 | dfr drop-nulls", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0_rolling_max".to_string(), - vec![ - Value::test_int(2), - Value::test_int(3), - Value::test_int(4), - Value::test_int(5), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let roll_type: Spanned = call.req(engine_state, stack, 0)?; - let window_size: i64 = call.req(engine_state, stack, 1)?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - if let DataType::Object(..) = series.dtype() { - return Err(ShellError::GenericError { - error: "Found object series".into(), - msg: "Series of type object cannot be used for rolling operation".into(), - span: Some(call.head), - help: None, - inner: vec![], - }); - } - - let roll_type = RollType::from_str(&roll_type.item, roll_type.span)?; - - let rolling_opts = RollingOptionsImpl { - window_size: Duration::new(window_size), - min_periods: window_size as usize, - weights: None, - center: false, - by: None, - closed_window: None, - tu: None, - tz: None, - fn_params: None, - }; - let res = match roll_type { - RollType::Max => series.rolling_max(rolling_opts), - RollType::Min => series.rolling_min(rolling_opts), - RollType::Sum => series.rolling_sum(rolling_opts), - RollType::Mean => series.rolling_mean(rolling_opts), - }; - - let mut res = res.map_err(|e| ShellError::GenericError { - error: "Error calculating rolling values".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let name = format!("{}_{}", series.name(), roll_type.to_str()); - res.rename(&name); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::eager::DropNulls; - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Rolling {}), Box::new(DropNulls {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs b/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs deleted file mode 100644 index 2f40cf0a45..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs +++ /dev/null @@ -1,115 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -use polars_plan::prelude::lit; - -#[derive(Clone)] -pub struct Shift; - -impl Command for Shift { - fn name(&self) -> &str { - "dfr shift" - } - - fn usage(&self) -> &str { - "Shifts the values by a given period." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("period", SyntaxShape::Int, "shift period") - .named( - "fill", - SyntaxShape::Any, - "Expression used to fill the null values (lazy df)", - Some('f'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe or lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Shifts the values by a given period", - example: "[1 2 2 3 3] | dfr into-df | dfr shift 2 | dfr drop-nulls", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(1), Value::test_int(2), Value::test_int(2)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuLazyFrame::can_downcast(&value) { - let df = NuLazyFrame::try_from_value(value)?; - command_lazy(engine_state, stack, call, df) - } else { - let df = NuDataFrame::try_from_value(value)?; - command_eager(engine_state, stack, call, df) - } - } -} - -fn command_eager( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let period: i64 = call.req(engine_state, stack, 0)?; - let series = df.as_series(call.head)?.shift(period); - - NuDataFrame::try_from_series(vec![series], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -fn command_lazy( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - lazy: NuLazyFrame, -) -> Result { - let shift: i64 = call.req(engine_state, stack, 0)?; - let fill: Option = call.get_flag(engine_state, stack, "fill")?; - - let lazy = lazy.into_polars(); - - let lazy: NuLazyFrame = match fill { - Some(fill) => { - let expr = NuExpression::try_from_value(fill)?.into_polars(); - lazy.shift_and_fill(lit(shift), expr).into() - } - None => lazy.shift(shift).into(), - }; - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) -} - -#[cfg(test)] -mod test { - use super::super::super::eager::DropNulls; - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Shift {}), Box::new(DropNulls {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/concatenate.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/concatenate.rs deleted file mode 100644 index d7589bd3b1..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/concatenate.rs +++ /dev/null @@ -1,113 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct Concatenate; - -impl Command for Concatenate { - fn name(&self) -> &str { - "dfr concatenate" - } - - fn usage(&self) -> &str { - "Concatenates strings with other array." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "other", - SyntaxShape::Any, - "Other array with string to be concatenated", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Concatenate string", - example: r#"let other = ([za xs cd] | dfr into-df); - [abc abc abc] | dfr into-df | dfr concatenate $other"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("abcza"), - Value::test_string("abcxs"), - Value::test_string("abccd"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let other: Value = call.req(engine_state, stack, 0)?; - let other_span = other.span(); - let other_df = NuDataFrame::try_from_value(other)?; - - let other_series = other_df.as_series(other_span)?; - let other_chunked = other_series.str().map_err(|e| ShellError::GenericError { - error: "The concatenate only with string columns".into(), - msg: e.to_string(), - span: Some(other_span), - help: None, - inner: vec![], - })?; - - let series = df.as_series(call.head)?; - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "The concatenate only with string columns".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let mut res = chunked.concat(other_chunked); - - res.rename(series.name()); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Concatenate {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/contains.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/contains.rs deleted file mode 100644 index 9c1d92681e..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/contains.rs +++ /dev/null @@ -1,106 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct Contains; - -impl Command for Contains { - fn name(&self) -> &str { - "dfr contains" - } - - fn usage(&self) -> &str { - "Checks if a pattern is contained in a string." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "pattern", - SyntaxShape::String, - "Regex pattern to be searched", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns boolean indicating if pattern was found", - example: "[abc acb acb] | dfr into-df | dfr contains ab", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_bool(true), - Value::test_bool(false), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let pattern: String = call.req(engine_state, stack, 0)?; - - let series = df.as_series(call.head)?; - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "The contains command only with string columns".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = chunked - .contains(&pattern, false) - .map_err(|e| ShellError::GenericError { - error: "Error searching in series".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Contains {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/mod.rs deleted file mode 100644 index f2fa19cbaf..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -mod concatenate; -mod contains; -mod replace; -mod replace_all; -mod str_lengths; -mod str_slice; -mod strftime; -mod to_lowercase; -mod to_uppercase; - -pub use concatenate::Concatenate; -pub use contains::Contains; -pub use replace::Replace; -pub use replace_all::ReplaceAll; -pub use str_lengths::StrLengths; -pub use str_slice::StrSlice; -pub use strftime::StrFTime; -pub use to_lowercase::ToLowerCase; -pub use to_uppercase::ToUpperCase; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/replace.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/replace.rs deleted file mode 100644 index d954e20b66..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/replace.rs +++ /dev/null @@ -1,120 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct Replace; - -impl Command for Replace { - fn name(&self) -> &str { - "dfr replace" - } - - fn usage(&self) -> &str { - "Replace the leftmost (sub)string by a regex pattern." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required_named( - "pattern", - SyntaxShape::String, - "Regex pattern to be matched", - Some('p'), - ) - .required_named( - "replace", - SyntaxShape::String, - "replacing string", - Some('r'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Replaces string", - example: "[abc abc abc] | dfr into-df | dfr replace --pattern ab --replace AB", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("ABc"), - Value::test_string("ABc"), - Value::test_string("ABc"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let pattern: String = call - .get_flag(engine_state, stack, "pattern")? - .expect("required value"); - let replace: String = call - .get_flag(engine_state, stack, "replace")? - .expect("required value"); - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "Error conversion to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let mut res = chunked - .replace(&pattern, &replace) - .map_err(|e| ShellError::GenericError { - error: "Error finding pattern other".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - res.rename(series.name()); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Replace {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/replace_all.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/replace_all.rs deleted file mode 100644 index f329cbca73..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/replace_all.rs +++ /dev/null @@ -1,121 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct ReplaceAll; - -impl Command for ReplaceAll { - fn name(&self) -> &str { - "dfr replace-all" - } - - fn usage(&self) -> &str { - "Replace all (sub)strings by a regex pattern." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required_named( - "pattern", - SyntaxShape::String, - "Regex pattern to be matched", - Some('p'), - ) - .required_named( - "replace", - SyntaxShape::String, - "replacing string", - Some('r'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Replaces string", - example: "[abac abac abac] | dfr into-df | dfr replace-all --pattern a --replace A", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("AbAc"), - Value::test_string("AbAc"), - Value::test_string("AbAc"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let pattern: String = call - .get_flag(engine_state, stack, "pattern")? - .expect("required value"); - let replace: String = call - .get_flag(engine_state, stack, "replace")? - .expect("required value"); - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "Error conversion to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let mut res = - chunked - .replace_all(&pattern, &replace) - .map_err(|e| ShellError::GenericError { - error: "Error finding pattern other".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - res.rename(series.name()); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ReplaceAll {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/str_lengths.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/str_lengths.rs deleted file mode 100644 index 6889cef387..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/str_lengths.rs +++ /dev/null @@ -1,87 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct StrLengths; - -impl Command for StrLengths { - fn name(&self) -> &str { - "dfr str-lengths" - } - - fn usage(&self) -> &str { - "Get lengths of all strings." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns string lengths", - example: "[a ab abc] | dfr into-df | dfr str-lengths", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(1), Value::test_int(2), Value::test_int(3)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-lengths command can only be used with string columns".into()), - inner: vec![], - })?; - - let res = chunked.as_ref().str_len_bytes().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(StrLengths {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/str_slice.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/str_slice.rs deleted file mode 100644 index 6a5c8364c2..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/str_slice.rs +++ /dev/null @@ -1,136 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::{ - prelude::{IntoSeries, NamedFrom, StringNameSpaceImpl}, - series::Series, -}; - -#[derive(Clone)] -pub struct StrSlice; - -impl Command for StrSlice { - fn name(&self) -> &str { - "dfr str-slice" - } - - fn usage(&self) -> &str { - "Slices the string from the start position until the selected length." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("start", SyntaxShape::Int, "start of slice") - .named("length", SyntaxShape::Int, "optional length", Some('l')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Creates slices from the strings", - example: "[abcded abc321 abc123] | dfr into-df | dfr str-slice 1 --length 2", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("bc"), - Value::test_string("bc"), - Value::test_string("bc"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates slices from the strings without length", - example: "[abcded abc321 abc123] | dfr into-df | dfr str-slice 1", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("bcded"), - Value::test_string("bc321"), - Value::test_string("bc123"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let start: i64 = call.req(engine_state, stack, 0)?; - let start = Series::new("", &[start]); - - let length: Option = call.get_flag(engine_state, stack, "length")?; - let length = match length { - Some(v) => Series::new("", &[v as u64]), - None => Series::new_null("", 1), - }; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-slice command can only be used with string columns".into()), - inner: vec![], - })?; - - let res = chunked - .str_slice(&start, &length) - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .with_name(series.name()); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(StrSlice {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/strftime.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/strftime.rs deleted file mode 100644 index 3cdfa84f8e..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/strftime.rs +++ /dev/null @@ -1,105 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::IntoSeries; - -#[derive(Clone)] -pub struct StrFTime; - -impl Command for StrFTime { - fn name(&self) -> &str { - "dfr strftime" - } - - fn usage(&self) -> &str { - "Formats date based on string rule." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("fmt", SyntaxShape::String, "Format rule") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Formats date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr strftime "%Y/%m/%d""#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("2020/08/04"), - Value::test_string("2020/08/04"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let fmt: String = call.req(engine_state, stack, 0)?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to date".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-slice command can only be used with string columns".into()), - inner: vec![], - })?; - - let res = casted - .strftime(&fmt) - .map_err(|e| ShellError::GenericError { - error: "Error formatting datetime".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(StrFTime {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/to_lowercase.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/to_lowercase.rs deleted file mode 100644 index 2340437e35..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/to_lowercase.rs +++ /dev/null @@ -1,92 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct ToLowerCase; - -impl Command for ToLowerCase { - fn name(&self) -> &str { - "dfr lowercase" - } - - fn usage(&self) -> &str { - "Lowercase the strings in the column." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Modifies strings to lowercase", - example: "[Abc aBc abC] | dfr into-df | dfr lowercase", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("abc"), - Value::test_string("abc"), - Value::test_string("abc"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-slice command can only be used with string columns".into()), - inner: vec![], - })?; - - let mut res = casted.to_lowercase(); - res.rename(series.name()); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ToLowerCase {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/to_uppercase.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/to_uppercase.rs deleted file mode 100644 index 23378f5dc3..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/to_uppercase.rs +++ /dev/null @@ -1,96 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct ToUpperCase; - -impl Command for ToUpperCase { - fn name(&self) -> &str { - "dfr uppercase" - } - - fn usage(&self) -> &str { - "Uppercase the strings in the column." - } - - fn search_terms(&self) -> Vec<&str> { - vec!["capitalize, caps, capital"] - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Modifies strings to uppercase", - example: "[Abc aBc abC] | dfr into-df | dfr uppercase", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("ABC"), - Value::test_string("ABC"), - Value::test_string("ABC"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-slice command can only be used with string columns".into()), - inner: vec![], - })?; - - let mut res = casted.to_uppercase(); - res.rename(series.name()); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ToUpperCase {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs b/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs deleted file mode 100644 index 1bc2e0dc1b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs +++ /dev/null @@ -1,146 +0,0 @@ -use crate::dataframe::{ - utils::extract_strings, - values::{Column, NuDataFrame, NuLazyFrame}, -}; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, UniqueKeepStrategy}; - -#[derive(Clone)] -pub struct Unique; - -impl Command for Unique { - fn name(&self) -> &str { - "dfr unique" - } - - fn usage(&self) -> &str { - "Returns unique values from a dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .named( - "subset", - SyntaxShape::Any, - "Subset of column(s) to use to maintain rows (lazy df)", - Some('s'), - ) - .switch( - "last", - "Keeps last unique value. Default keeps first value (lazy df)", - Some('l'), - ) - .switch( - "maintain-order", - "Keep the same order as the original DataFrame (lazy df)", - Some('k'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe or lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Returns unique values from a series", - example: "[2 2 2 2 2] | dfr into-df | dfr unique", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new("0".to_string(), vec![Value::test_int(2)])], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates a is unique expression from a column", - example: "col a | unique", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuLazyFrame::can_downcast(&value) { - let df = NuLazyFrame::try_from_value(value)?; - command_lazy(engine_state, stack, call, df) - } else { - let df = NuDataFrame::try_from_value(value)?; - command_eager(engine_state, stack, call, df) - } - } -} - -fn command_eager( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let series = df.as_series(call.head)?; - - let res = series.unique().map_err(|e| ShellError::GenericError { - error: "Error calculating unique values".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-slice command can only be used with string columns".into()), - inner: vec![], - })?; - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -fn command_lazy( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - lazy: NuLazyFrame, -) -> Result { - let last = call.has_flag(engine_state, stack, "last")?; - let maintain = call.has_flag(engine_state, stack, "maintain-order")?; - - let subset: Option = call.get_flag(engine_state, stack, "subset")?; - let subset = match subset { - Some(value) => Some(extract_strings(value)?), - None => None, - }; - - let strategy = if last { - UniqueKeepStrategy::Last - } else { - UniqueKeepStrategy::First - }; - - let lazy = lazy.into_polars(); - let lazy: NuLazyFrame = if maintain { - lazy.unique(subset, strategy).into() - } else { - lazy.unique_stable(subset, strategy).into() - }; - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Unique {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/value_counts.rs b/crates/nu-cmd-dataframe/src/dataframe/series/value_counts.rs deleted file mode 100644 index 87d3b42b3a..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/value_counts.rs +++ /dev/null @@ -1,95 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::SeriesMethods; - -#[derive(Clone)] -pub struct ValueCount; - -impl Command for ValueCount { - fn name(&self) -> &str { - "dfr value-counts" - } - - fn usage(&self) -> &str { - "Returns a dataframe with the counts for unique values in series." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Calculates value counts", - example: "[5 5 5 5 6 6] | dfr into-df | dfr value-counts", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "0".to_string(), - vec![Value::test_int(5), Value::test_int(6)], - ), - Column::new( - "count".to_string(), - vec![Value::test_int(4), Value::test_int(2)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let res = series - .value_counts(false, false) - .map_err(|e| ShellError::GenericError { - error: "Error calculating value counts values".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-slice command can only be used with string columns".into()), - inner: vec![], - })?; - - Ok(PipelineData::Value( - NuDataFrame::dataframe_into_value(res, call.head), - None, - )) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ValueCount {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/stub.rs b/crates/nu-cmd-dataframe/src/dataframe/stub.rs deleted file mode 100644 index 2d8cfde423..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/stub.rs +++ /dev/null @@ -1,44 +0,0 @@ -use nu_engine::{command_prelude::*, get_full_help}; - -#[derive(Clone)] -pub struct Dfr; - -impl Command for Dfr { - fn name(&self) -> &str { - "dfr" - } - - fn usage(&self) -> &str { - "Operate with data in a dataframe format." - } - - fn signature(&self) -> nu_protocol::Signature { - Signature::build("dfr") - .category(Category::Custom("dataframe".into())) - .input_output_types(vec![(Type::Nothing, Type::String)]) - } - - fn extra_usage(&self) -> &str { - "You must use one of the following subcommands. Using this command as-is will only produce this help message." - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - Ok(Value::string( - get_full_help( - &Dfr.signature(), - &Dfr.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs b/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs deleted file mode 100644 index 39c30be9dd..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs +++ /dev/null @@ -1,98 +0,0 @@ -use super::{ - eager::{SchemaDF, ToDataFrame}, - expressions::ExprCol, - lazy::{LazyCollect, LazyFillNull, ToLazyFrame}, -}; -use nu_cmd_lang::Let; -use nu_engine::{command_prelude::*, eval_block}; -use nu_parser::parse; -use nu_protocol::{debugger::WithoutDebug, engine::StateWorkingSet}; - -pub fn test_dataframe(cmds: Vec>) { - if cmds.is_empty() { - panic!("Empty commands vector") - } - - // The first element in the cmds vector must be the one tested - let examples = cmds[0].examples(); - let mut engine_state = build_test_engine_state(cmds.clone()); - - for example in examples { - test_dataframe_example(&mut engine_state, &example); - } -} - -pub fn build_test_engine_state(cmds: Vec>) -> Box { - let mut engine_state = Box::new(EngineState::new()); - - let delta = { - // Base functions that are needed for testing - // Try to keep this working set small to keep tests running as fast as possible - let mut working_set = StateWorkingSet::new(&engine_state); - working_set.add_decl(Box::new(Let)); - working_set.add_decl(Box::new(ToDataFrame)); - working_set.add_decl(Box::new(ToLazyFrame)); - working_set.add_decl(Box::new(LazyCollect)); - working_set.add_decl(Box::new(ExprCol)); - working_set.add_decl(Box::new(SchemaDF)); - working_set.add_decl(Box::new(LazyFillNull)); - - // Adding the command that is being tested to the working set - for cmd in cmds.clone() { - working_set.add_decl(cmd); - } - - working_set.render() - }; - - engine_state - .merge_delta(delta) - .expect("Error merging delta"); - - engine_state -} - -pub fn test_dataframe_example(engine_state: &mut Box, example: &Example) { - // Skip tests that don't have results to compare to - if example.result.is_none() { - return; - } - - let start = std::time::Instant::now(); - - let (block, delta) = { - let mut working_set = StateWorkingSet::new(engine_state); - let output = parse(&mut working_set, None, example.example.as_bytes(), false); - - if let Some(err) = working_set.parse_errors.first() { - panic!("test parse error in `{}`: {:?}", example.example, err) - } - - (output, working_set.render()) - }; - - engine_state - .merge_delta(delta) - .expect("Error merging delta"); - - let mut stack = Stack::new().capture(); - - let result = - eval_block::(engine_state, &mut stack, &block, PipelineData::empty()) - .unwrap_or_else(|err| panic!("test eval error in `{}`: {:?}", example.example, err)) - .into_value(Span::test_data()) - .expect("ok value"); - - println!("input: {}", example.example); - println!("result: {result:?}"); - println!("done: {:?}", start.elapsed()); - - // Note. Value implements PartialEq for Bool, Int, Float, String and Block - // If the command you are testing requires to compare another case, then - // you need to define its equality in the Value struct - if let Some(expected) = example.result.clone() { - if result != expected { - panic!("the example result is different to expected value: {result:?} != {expected:?}") - } - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/utils.rs b/crates/nu-cmd-dataframe/src/dataframe/utils.rs deleted file mode 100644 index db99d550a9..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/utils.rs +++ /dev/null @@ -1,16 +0,0 @@ -use nu_protocol::{FromValue, ShellError, Value}; - -pub fn extract_strings(value: Value) -> Result, ShellError> { - let span = value.span(); - match ( - ::from_value(value.clone()), - as FromValue>::from_value(value), - ) { - (Ok(col), Err(_)) => Ok(vec![col]), - (Err(_), Ok(cols)) => Ok(cols), - _ => Err(ShellError::IncompatibleParametersSingle { - msg: "Expected a string or list of strings".into(), - span, - }), - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/mod.rs deleted file mode 100644 index eaed15aa4b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/mod.rs +++ /dev/null @@ -1,14 +0,0 @@ -mod nu_dataframe; -mod nu_expression; -mod nu_lazyframe; -mod nu_lazygroupby; -mod nu_schema; -mod nu_when; -pub mod utils; - -pub use nu_dataframe::{Axis, Column, NuDataFrame}; -pub use nu_expression::NuExpression; -pub use nu_lazyframe::NuLazyFrame; -pub use nu_lazygroupby::NuLazyGroupBy; -pub use nu_schema::{str_to_dtype, NuSchema}; -pub use nu_when::NuWhen; diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/between_values.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/between_values.rs deleted file mode 100644 index 74a484825a..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/between_values.rs +++ /dev/null @@ -1,884 +0,0 @@ -use super::{operations::Axis, NuDataFrame}; -use nu_protocol::{ - ast::{Boolean, Comparison, Math, Operator}, - ShellError, Span, Spanned, Value, -}; -use num::Zero; -use polars::prelude::{ - BooleanType, ChunkCompare, ChunkedArray, DataType, Float64Type, Int64Type, IntoSeries, - NumOpsDispatchChecked, PolarsError, Series, StringNameSpaceImpl, -}; -use std::ops::{Add, BitAnd, BitOr, Div, Mul, Sub}; - -pub(super) fn between_dataframes( - operator: Spanned, - left: &Value, - lhs: &NuDataFrame, - right: &Value, - rhs: &NuDataFrame, -) -> Result { - let operation_span = Span::merge(left.span(), right.span()); - match operator.item { - Operator::Math(Math::Plus) => match lhs.append_df(rhs, Axis::Row, operation_span) { - Ok(df) => Ok(df.into_value(operation_span)), - Err(e) => Err(e), - }, - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - } -} - -pub(super) fn compute_between_series( - operator: Spanned, - left: &Value, - lhs: &Series, - right: &Value, - rhs: &Series, -) -> Result { - let operation_span = Span::merge(left.span(), right.span()); - match operator.item { - Operator::Math(Math::Plus) => { - let mut res = lhs + rhs; - let name = format!("sum_{}_{}", lhs.name(), rhs.name()); - res.rename(&name); - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Math(Math::Minus) => { - let mut res = lhs - rhs; - let name = format!("sub_{}_{}", lhs.name(), rhs.name()); - res.rename(&name); - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Math(Math::Multiply) => { - let mut res = lhs * rhs; - let name = format!("mul_{}_{}", lhs.name(), rhs.name()); - res.rename(&name); - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Math(Math::Divide) => { - let res = lhs.checked_div(rhs); - match res { - Ok(mut res) => { - let name = format!("div_{}_{}", lhs.name(), rhs.name()); - res.rename(&name); - NuDataFrame::series_to_value(res, operation_span) - } - Err(e) => Err(ShellError::GenericError { - error: "Division error".into(), - msg: e.to_string(), - span: Some(right.span()), - help: None, - inner: vec![], - }), - } - } - Operator::Comparison(Comparison::Equal) => { - let name = format!("eq_{}_{}", lhs.name(), rhs.name()); - let res = compare_series(lhs, rhs, name.as_str(), right.span(), Series::equal)?; - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Comparison(Comparison::NotEqual) => { - let name = format!("neq_{}_{}", lhs.name(), rhs.name()); - let res = compare_series(lhs, rhs, name.as_str(), right.span(), Series::not_equal)?; - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Comparison(Comparison::LessThan) => { - let name = format!("lt_{}_{}", lhs.name(), rhs.name()); - let res = compare_series(lhs, rhs, name.as_str(), right.span(), Series::lt)?; - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Comparison(Comparison::LessThanOrEqual) => { - let name = format!("lte_{}_{}", lhs.name(), rhs.name()); - let res = compare_series(lhs, rhs, name.as_str(), right.span(), Series::lt_eq)?; - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Comparison(Comparison::GreaterThan) => { - let name = format!("gt_{}_{}", lhs.name(), rhs.name()); - let res = compare_series(lhs, rhs, name.as_str(), right.span(), Series::gt)?; - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Comparison(Comparison::GreaterThanOrEqual) => { - let name = format!("gte_{}_{}", lhs.name(), rhs.name()); - let res = compare_series(lhs, rhs, name.as_str(), right.span(), Series::gt_eq)?; - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Boolean(Boolean::And) => match lhs.dtype() { - DataType::Boolean => { - let lhs_cast = lhs.bool(); - let rhs_cast = rhs.bool(); - - match (lhs_cast, rhs_cast) { - (Ok(l), Ok(r)) => { - let mut res = l.bitand(r).into_series(); - let name = format!("and_{}_{}", lhs.name(), rhs.name()); - res.rename(&name); - NuDataFrame::series_to_value(res, operation_span) - } - _ => Err(ShellError::GenericError { - error: "Incompatible types".into(), - msg: "unable to cast to boolean".into(), - span: Some(right.span()), - help: None, - inner: vec![], - }), - } - } - _ => Err(ShellError::IncompatibleParametersSingle { - msg: format!( - "Operation {} can only be done with boolean values", - operator.item - ), - span: operation_span, - }), - }, - Operator::Boolean(Boolean::Or) => match lhs.dtype() { - DataType::Boolean => { - let lhs_cast = lhs.bool(); - let rhs_cast = rhs.bool(); - - match (lhs_cast, rhs_cast) { - (Ok(l), Ok(r)) => { - let mut res = l.bitor(r).into_series(); - let name = format!("or_{}_{}", lhs.name(), rhs.name()); - res.rename(&name); - NuDataFrame::series_to_value(res, operation_span) - } - _ => Err(ShellError::GenericError { - error: "Incompatible types".into(), - msg: "unable to cast to boolean".into(), - span: Some(right.span()), - help: None, - inner: vec![], - }), - } - } - _ => Err(ShellError::IncompatibleParametersSingle { - msg: format!( - "Operation {} can only be done with boolean values", - operator.item - ), - span: operation_span, - }), - }, - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - } -} - -fn compare_series<'s, F>( - lhs: &'s Series, - rhs: &'s Series, - name: &'s str, - span: Span, - f: F, -) -> Result -where - F: Fn(&'s Series, &'s Series) -> Result, PolarsError>, -{ - let mut res = f(lhs, rhs) - .map_err(|e| ShellError::GenericError { - error: "Equality error".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })? - .into_series(); - - res.rename(name); - Ok(res) -} - -pub(super) fn compute_series_single_value( - operator: Spanned, - left: &Value, - lhs: &NuDataFrame, - right: &Value, -) -> Result { - if !lhs.is_series() { - return Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }); - } - - let lhs_span = left.span(); - let lhs = lhs.as_series(lhs_span)?; - - match operator.item { - Operator::Math(Math::Plus) => match &right { - Value::Int { val, .. } => { - compute_series_i64(&lhs, *val, >::add, lhs_span) - } - Value::Float { val, .. } => { - compute_series_float(&lhs, *val, >::add, lhs_span) - } - Value::String { val, .. } => add_string_to_series(&lhs, val, lhs_span), - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Math(Math::Minus) => match &right { - Value::Int { val, .. } => { - compute_series_i64(&lhs, *val, >::sub, lhs_span) - } - Value::Float { val, .. } => { - compute_series_float(&lhs, *val, >::sub, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Math(Math::Multiply) => match &right { - Value::Int { val, .. } => { - compute_series_i64(&lhs, *val, >::mul, lhs_span) - } - Value::Float { val, .. } => { - compute_series_float(&lhs, *val, >::mul, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Math(Math::Divide) => { - let span = right.span(); - match &right { - Value::Int { val, .. } => { - if *val == 0 { - Err(ShellError::DivisionByZero { span }) - } else { - compute_series_i64(&lhs, *val, >::div, lhs_span) - } - } - Value::Float { val, .. } => { - if val.is_zero() { - Err(ShellError::DivisionByZero { span }) - } else { - compute_series_float(&lhs, *val, >::div, lhs_span) - } - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - } - } - Operator::Comparison(Comparison::Equal) => match &right { - Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::equal, lhs_span), - Value::Float { val, .. } => { - compare_series_float(&lhs, *val, ChunkedArray::equal, lhs_span) - } - Value::String { val, .. } => { - let equal_pattern = format!("^{}$", fancy_regex::escape(val)); - contains_series_pat(&lhs, &equal_pattern, lhs_span) - } - Value::Date { val, .. } => { - compare_series_i64(&lhs, val.timestamp_millis(), ChunkedArray::equal, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::NotEqual) => match &right { - Value::Int { val, .. } => { - compare_series_i64(&lhs, *val, ChunkedArray::not_equal, lhs_span) - } - Value::Float { val, .. } => { - compare_series_float(&lhs, *val, ChunkedArray::not_equal, lhs_span) - } - Value::Date { val, .. } => compare_series_i64( - &lhs, - val.timestamp_millis(), - ChunkedArray::not_equal, - lhs_span, - ), - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::LessThan) => match &right { - Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::lt, lhs_span), - Value::Float { val, .. } => { - compare_series_float(&lhs, *val, ChunkedArray::lt, lhs_span) - } - Value::Date { val, .. } => { - compare_series_i64(&lhs, val.timestamp_millis(), ChunkedArray::lt, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::LessThanOrEqual) => match &right { - Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::lt_eq, lhs_span), - Value::Float { val, .. } => { - compare_series_float(&lhs, *val, ChunkedArray::lt_eq, lhs_span) - } - Value::Date { val, .. } => { - compare_series_i64(&lhs, val.timestamp_millis(), ChunkedArray::lt_eq, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::GreaterThan) => match &right { - Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::gt, lhs_span), - Value::Float { val, .. } => { - compare_series_float(&lhs, *val, ChunkedArray::gt, lhs_span) - } - Value::Date { val, .. } => { - compare_series_i64(&lhs, val.timestamp_millis(), ChunkedArray::gt, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::GreaterThanOrEqual) => match &right { - Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::gt_eq, lhs_span), - Value::Float { val, .. } => { - compare_series_float(&lhs, *val, ChunkedArray::gt_eq, lhs_span) - } - Value::Date { val, .. } => { - compare_series_i64(&lhs, val.timestamp_millis(), ChunkedArray::gt_eq, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - // TODO: update this to do a regex match instead of a simple contains? - Operator::Comparison(Comparison::RegexMatch) => match &right { - Value::String { val, .. } => contains_series_pat(&lhs, val, lhs_span), - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::StartsWith) => match &right { - Value::String { val, .. } => { - let starts_with_pattern = format!("^{}", fancy_regex::escape(val)); - contains_series_pat(&lhs, &starts_with_pattern, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::EndsWith) => match &right { - Value::String { val, .. } => { - let ends_with_pattern = format!("{}$", fancy_regex::escape(val)); - contains_series_pat(&lhs, &ends_with_pattern, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - } -} - -fn compute_series_i64(series: &Series, val: i64, f: F, span: Span) -> Result -where - F: Fn(ChunkedArray, i64) -> ChunkedArray, -{ - match series.dtype() { - DataType::UInt32 | DataType::Int32 | DataType::UInt64 => { - let to_i64 = series.cast(&DataType::Int64); - - match to_i64 { - Ok(series) => { - let casted = series.i64(); - compute_casted_i64(casted, val, f, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to i64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - DataType::Int64 => { - let casted = series.i64(); - compute_casted_i64(casted, val, f, span) - } - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: format!( - "Series of type {} can not be used for operations with an i64 value", - series.dtype() - ), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compute_casted_i64( - casted: Result<&ChunkedArray, PolarsError>, - val: i64, - f: F, - span: Span, -) -> Result -where - F: Fn(ChunkedArray, i64) -> ChunkedArray, -{ - match casted { - Ok(casted) => { - let res = f(casted.clone(), val); - let res = res.into_series(); - NuDataFrame::series_to_value(res, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to i64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compute_series_float(series: &Series, val: f64, f: F, span: Span) -> Result -where - F: Fn(ChunkedArray, f64) -> ChunkedArray, -{ - match series.dtype() { - DataType::Float32 => { - let to_f64 = series.cast(&DataType::Float64); - - match to_f64 { - Ok(series) => { - let casted = series.f64(); - compute_casted_f64(casted, val, f, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - DataType::Float64 => { - let casted = series.f64(); - compute_casted_f64(casted, val, f, span) - } - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: format!( - "Series of type {} can not be used for operations with a float value", - series.dtype() - ), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compute_casted_f64( - casted: Result<&ChunkedArray, PolarsError>, - val: f64, - f: F, - span: Span, -) -> Result -where - F: Fn(ChunkedArray, f64) -> ChunkedArray, -{ - match casted { - Ok(casted) => { - let res = f(casted.clone(), val); - let res = res.into_series(); - NuDataFrame::series_to_value(res, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compare_series_i64(series: &Series, val: i64, f: F, span: Span) -> Result -where - F: Fn(&ChunkedArray, i64) -> ChunkedArray, -{ - match series.dtype() { - DataType::UInt32 | DataType::Int32 | DataType::UInt64 | DataType::Datetime(_, _) => { - let to_i64 = series.cast(&DataType::Int64); - - match to_i64 { - Ok(series) => { - let casted = series.i64(); - compare_casted_i64(casted, val, f, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - DataType::Date => { - let to_i64 = series.cast(&DataType::Int64); - - match to_i64 { - Ok(series) => { - let nanosecs_per_day: i64 = 24 * 60 * 60 * 1_000_000_000; - let casted = series - .i64() - .map(|chunked| chunked.mul(nanosecs_per_day)) - .expect("already checked for casting"); - compare_casted_i64(Ok(&casted), val, f, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - DataType::Int64 => { - let casted = series.i64(); - compare_casted_i64(casted, val, f, span) - } - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: format!( - "Series of type {} can not be used for operations with an i64 value", - series.dtype() - ), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compare_casted_i64( - casted: Result<&ChunkedArray, PolarsError>, - val: i64, - f: F, - span: Span, -) -> Result -where - F: Fn(&ChunkedArray, i64) -> ChunkedArray, -{ - match casted { - Ok(casted) => { - let res = f(casted, val); - let res = res.into_series(); - NuDataFrame::series_to_value(res, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to i64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compare_series_float(series: &Series, val: f64, f: F, span: Span) -> Result -where - F: Fn(&ChunkedArray, f64) -> ChunkedArray, -{ - match series.dtype() { - DataType::Float32 => { - let to_f64 = series.cast(&DataType::Float64); - - match to_f64 { - Ok(series) => { - let casted = series.f64(); - compare_casted_f64(casted, val, f, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to i64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - DataType::Float64 => { - let casted = series.f64(); - compare_casted_f64(casted, val, f, span) - } - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: format!( - "Series of type {} can not be used for operations with a float value", - series.dtype() - ), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compare_casted_f64( - casted: Result<&ChunkedArray, PolarsError>, - val: f64, - f: F, - span: Span, -) -> Result -where - F: Fn(&ChunkedArray, f64) -> ChunkedArray, -{ - match casted { - Ok(casted) => { - let res = f(casted, val); - let res = res.into_series(); - NuDataFrame::series_to_value(res, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn contains_series_pat(series: &Series, pat: &str, span: Span) -> Result { - let casted = series.str(); - match casted { - Ok(casted) => { - let res = casted.contains(pat, false); - - match res { - Ok(res) => { - let res = res.into_series(); - NuDataFrame::series_to_value(res, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Error using contains".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to string".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn add_string_to_series(series: &Series, pat: &str, span: Span) -> Result { - let casted = series.str(); - match casted { - Ok(casted) => { - let res = casted + pat; - let res = res.into_series(); - - NuDataFrame::series_to_value(res, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to string".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -#[cfg(test)] -mod test { - use super::*; - use nu_protocol::Span; - use polars::{prelude::NamedFrom, series::Series}; - - use crate::dataframe::values::NuDataFrame; - - #[test] - fn test_compute_between_series_comparisons() { - let series = Series::new("c", &[1, 2]); - let df = NuDataFrame::try_from_series(vec![series], Span::test_data()) - .expect("should be able to create a simple dataframe"); - - let c0 = df - .column("c", Span::test_data()) - .expect("should be able to get column c"); - - let c0_series = c0 - .as_series(Span::test_data()) - .expect("should be able to get series"); - - let c0_value = c0.into_value(Span::test_data()); - - let c1 = df - .column("c", Span::test_data()) - .expect("should be able to get column c"); - - let c1_series = c1 - .as_series(Span::test_data()) - .expect("should be able to get series"); - - let c1_value = c1.into_value(Span::test_data()); - - let op = Spanned { - item: Operator::Comparison(Comparison::NotEqual), - span: Span::test_data(), - }; - let result = compute_between_series(op, &c0_value, &c0_series, &c1_value, &c1_series) - .expect("compare should not fail"); - let result = NuDataFrame::try_from_value(result) - .expect("should be able to create a dataframe from a value"); - let result = result - .as_series(Span::test_data()) - .expect("should be convert to a series"); - assert_eq!(result, Series::new("neq_c_c", &[false, false])); - - let op = Spanned { - item: Operator::Comparison(Comparison::Equal), - span: Span::test_data(), - }; - let result = compute_between_series(op, &c0_value, &c0_series, &c1_value, &c1_series) - .expect("compare should not fail"); - let result = NuDataFrame::try_from_value(result) - .expect("should be able to create a dataframe from a value"); - let result = result - .as_series(Span::test_data()) - .expect("should be convert to a series"); - assert_eq!(result, Series::new("eq_c_c", &[true, true])); - - let op = Spanned { - item: Operator::Comparison(Comparison::LessThan), - span: Span::test_data(), - }; - let result = compute_between_series(op, &c0_value, &c0_series, &c1_value, &c1_series) - .expect("compare should not fail"); - let result = NuDataFrame::try_from_value(result) - .expect("should be able to create a dataframe from a value"); - let result = result - .as_series(Span::test_data()) - .expect("should be convert to a series"); - assert_eq!(result, Series::new("lt_c_c", &[false, false])); - - let op = Spanned { - item: Operator::Comparison(Comparison::LessThanOrEqual), - span: Span::test_data(), - }; - let result = compute_between_series(op, &c0_value, &c0_series, &c1_value, &c1_series) - .expect("compare should not fail"); - let result = NuDataFrame::try_from_value(result) - .expect("should be able to create a dataframe from a value"); - let result = result - .as_series(Span::test_data()) - .expect("should be convert to a series"); - assert_eq!(result, Series::new("lte_c_c", &[true, true])); - - let op = Spanned { - item: Operator::Comparison(Comparison::GreaterThan), - span: Span::test_data(), - }; - let result = compute_between_series(op, &c0_value, &c0_series, &c1_value, &c1_series) - .expect("compare should not fail"); - let result = NuDataFrame::try_from_value(result) - .expect("should be able to create a dataframe from a value"); - let result = result - .as_series(Span::test_data()) - .expect("should be convert to a series"); - assert_eq!(result, Series::new("gt_c_c", &[false, false])); - - let op = Spanned { - item: Operator::Comparison(Comparison::GreaterThanOrEqual), - span: Span::test_data(), - }; - let result = compute_between_series(op, &c0_value, &c0_series, &c1_value, &c1_series) - .expect("compare should not fail"); - let result = NuDataFrame::try_from_value(result) - .expect("should be able to create a dataframe from a value"); - let result = result - .as_series(Span::test_data()) - .expect("should be convert to a series"); - assert_eq!(result, Series::new("gte_c_c", &[true, true])); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/conversion.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/conversion.rs deleted file mode 100644 index 7ab339d78d..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/conversion.rs +++ /dev/null @@ -1,1435 +0,0 @@ -use super::{DataFrameValue, NuDataFrame, NuSchema}; -use chrono::{DateTime, Duration, FixedOffset, NaiveTime, TimeZone, Utc}; -use chrono_tz::Tz; -use indexmap::map::{Entry, IndexMap}; -use nu_protocol::{Record, ShellError, Span, Value}; -use polars::{ - chunked_array::{ - builder::AnonymousOwnedListBuilder, object::builder::ObjectChunkedBuilder, ChunkedArray, - }, - datatypes::AnyValue, - export::arrow::Either, - prelude::{ - DataFrame, DataType, DatetimeChunked, Float32Type, Float64Type, Int16Type, Int32Type, - Int64Type, Int8Type, IntoSeries, ListBooleanChunkedBuilder, ListBuilderTrait, - ListPrimitiveChunkedBuilder, ListStringChunkedBuilder, ListType, NamedFrom, - NewChunkedArray, ObjectType, Schema, Series, StructChunked, TemporalMethods, TimeUnit, - UInt16Type, UInt32Type, UInt64Type, UInt8Type, - }, -}; -use std::ops::{Deref, DerefMut}; - -const NANOS_PER_DAY: i64 = 86_400_000_000_000; - -// The values capacity is for the size of an vec. -// Since this is impossible to determine without traversing every value -// I just picked one. Since this is for converting back and forth -// between nushell tables the values shouldn't be too extremely large for -// practical reasons (~ a few thousand rows). -const VALUES_CAPACITY: usize = 10; - -macro_rules! value_to_primitive { - ($value:ident, u8) => { - $value.as_i64().map(|v| v as u8) - }; - ($value:ident, u16) => { - $value.as_i64().map(|v| v as u16) - }; - ($value:ident, u32) => { - $value.as_i64().map(|v| v as u32) - }; - ($value:ident, u64) => { - $value.as_i64().map(|v| v as u64) - }; - ($value:ident, i8) => { - $value.as_i64().map(|v| v as i8) - }; - ($value:ident, i16) => { - $value.as_i64().map(|v| v as i16) - }; - ($value:ident, i32) => { - $value.as_i64().map(|v| v as i32) - }; - ($value:ident, i64) => { - $value.as_i64() - }; - ($value:ident, f32) => { - $value.as_f64().map(|v| v as f32) - }; - ($value:ident, f64) => { - $value.as_f64() - }; -} - -#[derive(Debug)] -pub struct Column { - name: String, - values: Vec, -} - -impl Column { - pub fn new(name: String, values: Vec) -> Self { - Self { name, values } - } - - pub fn new_empty(name: String) -> Self { - Self { - name, - values: Vec::new(), - } - } - - pub fn name(&self) -> &str { - self.name.as_str() - } -} - -impl IntoIterator for Column { - type Item = Value; - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.values.into_iter() - } -} - -impl Deref for Column { - type Target = Vec; - - fn deref(&self) -> &Self::Target { - &self.values - } -} - -impl DerefMut for Column { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.values - } -} - -#[derive(Debug)] -pub struct TypedColumn { - column: Column, - column_type: Option, -} - -impl TypedColumn { - fn new_empty(name: String) -> Self { - Self { - column: Column::new_empty(name), - column_type: None, - } - } -} - -impl Deref for TypedColumn { - type Target = Column; - - fn deref(&self) -> &Self::Target { - &self.column - } -} - -impl DerefMut for TypedColumn { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.column - } -} - -pub type ColumnMap = IndexMap; - -pub fn create_column( - series: &Series, - from_row: usize, - to_row: usize, - span: Span, -) -> Result { - let size = to_row - from_row; - let values = series_to_values(series, Some(from_row), Some(size), span)?; - Ok(Column::new(series.name().into(), values)) -} - -// Adds a separator to the vector of values using the column names from the -// dataframe to create the Values Row -pub fn add_separator(values: &mut Vec, df: &DataFrame, span: Span) { - let mut record = Record::new(); - - record.push("index", Value::string("...", span)); - - for name in df.get_column_names() { - record.push(name, Value::string("...", span)) - } - - values.push(Value::record(record, span)); -} - -// Inserting the values found in a Value::List or Value::Record -pub fn insert_record( - column_values: &mut ColumnMap, - record: Record, - maybe_schema: &Option, -) -> Result<(), ShellError> { - for (col, value) in record { - insert_value(value, col, column_values, maybe_schema)?; - } - - Ok(()) -} - -pub fn insert_value( - value: Value, - key: String, - column_values: &mut ColumnMap, - maybe_schema: &Option, -) -> Result<(), ShellError> { - let col_val = match column_values.entry(key.clone()) { - Entry::Vacant(entry) => entry.insert(TypedColumn::new_empty(key.clone())), - Entry::Occupied(entry) => entry.into_mut(), - }; - - // Checking that the type for the value is the same - // for the previous value in the column - if col_val.values.is_empty() { - if let Some(schema) = maybe_schema { - if let Some(field) = schema.schema.get_field(&key) { - col_val.column_type = Some(field.data_type().clone()); - } - } - - if col_val.column_type.is_none() { - col_val.column_type = Some(value_to_data_type(&value)); - } - - col_val.values.push(value); - } else { - let prev_value = &col_val.values[col_val.values.len() - 1]; - - match (&prev_value, &value) { - (Value::Int { .. }, Value::Int { .. }) - | (Value::Float { .. }, Value::Float { .. }) - | (Value::String { .. }, Value::String { .. }) - | (Value::Bool { .. }, Value::Bool { .. }) - | (Value::Date { .. }, Value::Date { .. }) - | (Value::Filesize { .. }, Value::Filesize { .. }) - | (Value::Duration { .. }, Value::Duration { .. }) => col_val.values.push(value), - (Value::List { .. }, _) => { - col_val.column_type = Some(value_to_data_type(&value)); - col_val.values.push(value); - } - _ => { - col_val.column_type = Some(DataType::Object("Value", None)); - col_val.values.push(value); - } - } - } - - Ok(()) -} - -fn value_to_data_type(value: &Value) -> DataType { - match &value { - Value::Int { .. } => DataType::Int64, - Value::Float { .. } => DataType::Float64, - Value::String { .. } => DataType::String, - Value::Bool { .. } => DataType::Boolean, - Value::Date { .. } => DataType::Date, - Value::Duration { .. } => DataType::Duration(TimeUnit::Nanoseconds), - Value::Filesize { .. } => DataType::Int64, - Value::List { vals, .. } => { - // We need to determined the type inside of the list. - // Since Value::List does not have any kind of - // type information, we need to look inside the list. - // This will cause errors if lists have inconsistent types. - // Basically, if a list column needs to be converted to dataframe, - // needs to have consistent types. - let list_type = vals - .iter() - .filter(|v| !matches!(v, Value::Nothing { .. })) - .map(value_to_data_type) - .nth(1) - .unwrap_or(DataType::Object("Value", None)); - - DataType::List(Box::new(list_type)) - } - _ => DataType::Object("Value", None), - } -} - -fn typed_column_to_series(name: &str, column: TypedColumn) -> Result { - if let Some(column_type) = &column.column_type { - match column_type { - DataType::Float32 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_f64().map(|v| v as f32)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Float64 => { - let series_values: Result, _> = - column.values.iter().map(|v| v.as_f64()).collect(); - Ok(Series::new(name, series_values?)) - } - DataType::UInt8 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as u8)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::UInt16 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as u16)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::UInt32 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as u32)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::UInt64 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as u64)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Int8 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as i8)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Int16 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as i16)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Int32 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as i32)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Int64 => { - let series_values: Result, _> = - column.values.iter().map(|v| v.as_i64()).collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Boolean => { - let series_values: Result, _> = - column.values.iter().map(|v| v.as_bool()).collect(); - Ok(Series::new(name, series_values?)) - } - DataType::String => { - let series_values: Result, _> = - column.values.iter().map(|v| v.coerce_string()).collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Object(_, _) => value_to_series(name, &column.values), - DataType::Duration(time_unit) => { - //todo - finish type conversion - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| nanos_from_timeunit(v, *time_unit))) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::List(list_type) => { - match input_type_list_to_series(name, list_type.as_ref(), &column.values) { - Ok(series) => Ok(series), - Err(_) => { - // An error case will occur when there are lists of mixed types. - // If this happens, fallback to object list - input_type_list_to_series( - name, - &DataType::Object("unknown", None), - &column.values, - ) - } - } - } - DataType::Date => { - let it = column.values.iter().map(|v| { - if let Value::Date { val, .. } = &v { - Some(val.timestamp_nanos_opt().unwrap_or_default()) - } else { - None - } - }); - - let res: DatetimeChunked = ChunkedArray::::from_iter_options(name, it) - .into_datetime(TimeUnit::Nanoseconds, None); - - Ok(res.into_series()) - } - DataType::Datetime(tu, maybe_tz) => { - let dates = column - .values - .iter() - .map(|v| { - if let Value::Date { val, .. } = &v { - // If there is a timezone specified, make sure - // the value is converted to it - Ok(maybe_tz - .as_ref() - .map(|tz| tz.parse::().map(|tz| val.with_timezone(&tz))) - .transpose() - .map_err(|e| ShellError::GenericError { - error: "Error parsing timezone".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })? - .and_then(|dt| dt.timestamp_nanos_opt()) - .map(|nanos| nanos_from_timeunit(nanos, *tu))) - } else { - Ok(None) - } - }) - .collect::>, ShellError>>()?; - - let res: DatetimeChunked = - ChunkedArray::::from_iter_options(name, dates.into_iter()) - .into_datetime(*tu, maybe_tz.clone()); - - Ok(res.into_series()) - } - DataType::Struct(fields) => { - let schema = Some(NuSchema::new(Schema::from_iter(fields.clone()))); - let mut structs: Vec = Vec::new(); - - for v in column.values.iter() { - let mut column_values: ColumnMap = IndexMap::new(); - let record = v.as_record()?; - insert_record(&mut column_values, record.clone(), &schema)?; - let df = from_parsed_columns(column_values)?; - structs.push(df.as_series(Span::unknown())?); - } - - let chunked = StructChunked::new(column.name(), structs.as_ref()).map_err(|e| { - ShellError::GenericError { - error: format!("Error creating struct: {e}"), - msg: "".into(), - span: None, - help: None, - inner: vec![], - } - })?; - Ok(chunked.into_series()) - } - _ => Err(ShellError::GenericError { - error: format!("Error creating dataframe: Unsupported type: {column_type:?}"), - msg: "".into(), - span: None, - help: None, - inner: vec![], - }), - } - } else { - Err(ShellError::GenericError { - error: "Passed a type column with no type".into(), - msg: "".into(), - span: None, - help: None, - inner: vec![], - }) - } -} - -// The ColumnMap has the parsed data from the StreamInput -// This data can be used to create a Series object that can initialize -// the dataframe based on the type of data that is found -pub fn from_parsed_columns(column_values: ColumnMap) -> Result { - let mut df_series: Vec = Vec::new(); - for (name, column) in column_values { - let series = typed_column_to_series(&name, column)?; - df_series.push(series); - } - - DataFrame::new(df_series) - .map(|df| NuDataFrame::new(false, df)) - .map_err(|e| ShellError::GenericError { - error: "Error creating dataframe".into(), - msg: e.to_string(), - span: None, - help: None, - inner: vec![], - }) -} - -fn value_to_series(name: &str, values: &[Value]) -> Result { - let mut builder = ObjectChunkedBuilder::::new(name, values.len()); - - for v in values { - builder.append_value(DataFrameValue::new(v.clone())); - } - - let res = builder.finish(); - Ok(res.into_series()) -} - -fn input_type_list_to_series( - name: &str, - data_type: &DataType, - values: &[Value], -) -> Result { - let inconsistent_error = |_| ShellError::GenericError { - error: format!( - "column {name} contains a list with inconsistent types: Expecting: {data_type:?}" - ), - msg: "".into(), - span: None, - help: None, - inner: vec![], - }; - - macro_rules! primitive_list_series { - ($list_type:ty, $vec_type:tt) => {{ - let mut builder = ListPrimitiveChunkedBuilder::<$list_type>::new( - name, - values.len(), - VALUES_CAPACITY, - data_type.clone(), - ); - - for v in values { - let value_list = v - .as_list()? - .iter() - .map(|v| value_to_primitive!(v, $vec_type)) - .collect::, _>>() - .map_err(inconsistent_error)?; - builder.append_iter_values(value_list.iter().copied()); - } - let res = builder.finish(); - Ok(res.into_series()) - }}; - } - - match *data_type { - // list of boolean values - DataType::Boolean => { - let mut builder = ListBooleanChunkedBuilder::new(name, values.len(), VALUES_CAPACITY); - for v in values { - let value_list = v - .as_list()? - .iter() - .map(|v| v.as_bool()) - .collect::, _>>() - .map_err(inconsistent_error)?; - builder.append_iter(value_list.iter().map(|v| Some(*v))); - } - let res = builder.finish(); - Ok(res.into_series()) - } - DataType::Duration(_) => primitive_list_series!(Int64Type, i64), - DataType::UInt8 => primitive_list_series!(UInt8Type, u8), - DataType::UInt16 => primitive_list_series!(UInt16Type, u16), - DataType::UInt32 => primitive_list_series!(UInt32Type, u32), - DataType::UInt64 => primitive_list_series!(UInt64Type, u64), - DataType::Int8 => primitive_list_series!(Int8Type, i8), - DataType::Int16 => primitive_list_series!(Int16Type, i16), - DataType::Int32 => primitive_list_series!(Int32Type, i32), - DataType::Int64 => primitive_list_series!(Int64Type, i64), - DataType::Float32 => primitive_list_series!(Float32Type, f32), - DataType::Float64 => primitive_list_series!(Float64Type, f64), - DataType::String => { - let mut builder = ListStringChunkedBuilder::new(name, values.len(), VALUES_CAPACITY); - for v in values { - let value_list = v - .as_list()? - .iter() - .map(|v| v.coerce_string()) - .collect::, _>>() - .map_err(inconsistent_error)?; - builder.append_values_iter(value_list.iter().map(AsRef::as_ref)); - } - let res = builder.finish(); - Ok(res.into_series()) - } - DataType::Date => { - let mut builder = AnonymousOwnedListBuilder::new( - name, - values.len(), - Some(DataType::Datetime(TimeUnit::Nanoseconds, None)), - ); - for (i, v) in values.iter().enumerate() { - let list_name = i.to_string(); - - let it = v.as_list()?.iter().map(|v| { - if let Value::Date { val, .. } = &v { - Some(val.timestamp_nanos_opt().unwrap_or_default()) - } else { - None - } - }); - let dt_chunked = ChunkedArray::::from_iter_options(&list_name, it) - .into_datetime(TimeUnit::Nanoseconds, None); - - builder - .append_series(&dt_chunked.into_series()) - .map_err(|e| ShellError::GenericError { - error: "Error appending to series".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })? - } - let res = builder.finish(); - Ok(res.into_series()) - } - DataType::List(ref sub_list_type) => { - Ok(input_type_list_to_series(name, sub_list_type, values)?) - } - // treat everything else as an object - _ => Ok(value_to_series(name, values)?), - } -} - -fn series_to_values( - series: &Series, - maybe_from_row: Option, - maybe_size: Option, - span: Span, -) -> Result, ShellError> { - match series.dtype() { - DataType::Null => { - let it = std::iter::repeat(Value::nothing(span)); - let values = if let Some(size) = maybe_size { - Either::Left(it.take(size)) - } else { - Either::Right(it) - } - .collect::>(); - - Ok(values) - } - DataType::UInt8 => { - let casted = series.u8().map_err(|e| ShellError::GenericError { - error: "Error casting column to u8".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::UInt16 => { - let casted = series.u16().map_err(|e| ShellError::GenericError { - error: "Error casting column to u16".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::UInt32 => { - let casted = series.u32().map_err(|e| ShellError::GenericError { - error: "Error casting column to u32".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::UInt64 => { - let casted = series.u64().map_err(|e| ShellError::GenericError { - error: "Error casting column to u64".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Int8 => { - let casted = series.i8().map_err(|e| ShellError::GenericError { - error: "Error casting column to i8".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Int16 => { - let casted = series.i16().map_err(|e| ShellError::GenericError { - error: "Error casting column to i16".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Int32 => { - let casted = series.i32().map_err(|e| ShellError::GenericError { - error: "Error casting column to i32".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Int64 => { - let casted = series.i64().map_err(|e| ShellError::GenericError { - error: "Error casting column to i64".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Float32 => { - let casted = series.f32().map_err(|e| ShellError::GenericError { - error: "Error casting column to f32".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::float(a as f64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Float64 => { - let casted = series.f64().map_err(|e| ShellError::GenericError { - error: "Error casting column to f64".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::float(a, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Boolean => { - let casted = series.bool().map_err(|e| ShellError::GenericError { - error: "Error casting column to bool".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::bool(a, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::String => { - let casted = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting column to string".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::string(a.to_string(), span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Object(x, _) => { - let casted = series - .as_any() - .downcast_ref::>>(); - - match casted { - None => Err(ShellError::GenericError { - error: "Error casting object from series".into(), - msg: "".into(), - span: None, - help: Some(format!("Object not supported for conversion: {x}")), - inner: vec![], - }), - Some(ca) => { - let it = ca.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) - { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => a.get_value(), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - } - } - DataType::List(x) => { - let casted = series.as_any().downcast_ref::>(); - match casted { - None => Err(ShellError::GenericError { - error: "Error casting list from series".into(), - msg: "".into(), - span: None, - help: Some(format!("List not supported for conversion: {x}")), - inner: vec![], - }), - Some(ca) => { - let it = ca.into_iter(); - if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|ca| { - let sublist: Vec = if let Some(ref s) = ca { - series_to_values(s, None, None, Span::unknown())? - } else { - // empty item - vec![] - }; - Ok(Value::list(sublist, span)) - }) - .collect::, ShellError>>() - } - } - } - DataType::Date => { - let casted = series.date().map_err(|e| ShellError::GenericError { - error: "Error casting column to date".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => { - let nanos = nanos_per_day(a); - let datetime = datetime_from_epoch_nanos(nanos, &None, span)?; - Ok(Value::date(datetime, span)) - } - None => Ok(Value::nothing(span)), - }) - .collect::, ShellError>>()?; - Ok(values) - } - DataType::Datetime(time_unit, tz) => { - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting column to datetime".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => { - // elapsed time in nano/micro/milliseconds since 1970-01-01 - let nanos = nanos_from_timeunit(a, *time_unit); - let datetime = datetime_from_epoch_nanos(nanos, tz, span)?; - Ok(Value::date(datetime, span)) - } - None => Ok(Value::nothing(span)), - }) - .collect::, ShellError>>()?; - Ok(values) - } - DataType::Struct(polar_fields) => { - let casted = series.struct_().map_err(|e| ShellError::GenericError { - error: "Error casting column to struct".into(), - msg: "".to_string(), - span: None, - help: Some(e.to_string()), - inner: Vec::new(), - })?; - let it = casted.into_iter(); - let values: Result, ShellError> = - if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|any_values| { - let record = polar_fields - .iter() - .zip(any_values) - .map(|(field, val)| { - any_value_to_value(val, span).map(|val| (field.name.to_string(), val)) - }) - .collect::>()?; - - Ok(Value::record(record, span)) - }) - .collect(); - values - } - DataType::Time => { - let casted = - series - .timestamp(TimeUnit::Nanoseconds) - .map_err(|e| ShellError::GenericError { - error: "Error casting column to time".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(nanoseconds) => Value::duration(nanoseconds, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - e => Err(ShellError::GenericError { - error: "Error creating Dataframe".into(), - msg: "".to_string(), - span: None, - help: Some(format!("Value not supported in nushell: {e}")), - inner: vec![], - }), - } -} - -fn any_value_to_value(any_value: &AnyValue, span: Span) -> Result { - match any_value { - AnyValue::Null => Ok(Value::nothing(span)), - AnyValue::Boolean(b) => Ok(Value::bool(*b, span)), - AnyValue::String(s) => Ok(Value::string(s.to_string(), span)), - AnyValue::UInt8(i) => Ok(Value::int(*i as i64, span)), - AnyValue::UInt16(i) => Ok(Value::int(*i as i64, span)), - AnyValue::UInt32(i) => Ok(Value::int(*i as i64, span)), - AnyValue::UInt64(i) => Ok(Value::int(*i as i64, span)), - AnyValue::Int8(i) => Ok(Value::int(*i as i64, span)), - AnyValue::Int16(i) => Ok(Value::int(*i as i64, span)), - AnyValue::Int32(i) => Ok(Value::int(*i as i64, span)), - AnyValue::Int64(i) => Ok(Value::int(*i, span)), - AnyValue::Float32(f) => Ok(Value::float(*f as f64, span)), - AnyValue::Float64(f) => Ok(Value::float(*f, span)), - AnyValue::Date(d) => { - let nanos = nanos_per_day(*d); - datetime_from_epoch_nanos(nanos, &None, span) - .map(|datetime| Value::date(datetime, span)) - } - AnyValue::Datetime(a, time_unit, tz) => { - let nanos = nanos_from_timeunit(*a, *time_unit); - datetime_from_epoch_nanos(nanos, tz, span).map(|datetime| Value::date(datetime, span)) - } - AnyValue::Duration(a, time_unit) => { - let nanos = match time_unit { - TimeUnit::Nanoseconds => *a, - TimeUnit::Microseconds => *a * 1_000, - TimeUnit::Milliseconds => *a * 1_000_000, - }; - Ok(Value::duration(nanos, span)) - } - // AnyValue::Time represents the current time since midnight. - // Unfortunately, there is no timezone related information. - // Given this, calculate the current date from UTC and add the time. - AnyValue::Time(nanos) => time_from_midnight(*nanos, span), - AnyValue::List(series) => { - series_to_values(series, None, None, span).map(|values| Value::list(values, span)) - } - AnyValue::Struct(_idx, _struct_array, _s_fields) => { - // This should convert to a StructOwned object. - let static_value = - any_value - .clone() - .into_static() - .map_err(|e| ShellError::GenericError { - error: "Cannot convert polars struct to static value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: Vec::new(), - })?; - any_value_to_value(&static_value, span) - } - AnyValue::StructOwned(struct_tuple) => { - let record = struct_tuple - .1 - .iter() - .zip(&struct_tuple.0) - .map(|(field, val)| { - any_value_to_value(val, span).map(|val| (field.name.to_string(), val)) - }) - .collect::>()?; - - Ok(Value::record(record, span)) - } - AnyValue::StringOwned(s) => Ok(Value::string(s.to_string(), span)), - AnyValue::Binary(bytes) => Ok(Value::binary(*bytes, span)), - AnyValue::BinaryOwned(bytes) => Ok(Value::binary(bytes.to_owned(), span)), - e => Err(ShellError::GenericError { - error: "Error creating Value".into(), - msg: "".to_string(), - span: None, - help: Some(format!("Value not supported in nushell: {e}")), - inner: Vec::new(), - }), - } -} - -fn nanos_per_day(days: i32) -> i64 { - days as i64 * NANOS_PER_DAY -} - -fn nanos_from_timeunit(a: i64, time_unit: TimeUnit) -> i64 { - a * match time_unit { - TimeUnit::Microseconds => 1_000, // Convert microseconds to nanoseconds - TimeUnit::Milliseconds => 1_000_000, // Convert milliseconds to nanoseconds - TimeUnit::Nanoseconds => 1, // Already in nanoseconds - } -} - -fn datetime_from_epoch_nanos( - nanos: i64, - timezone: &Option, - span: Span, -) -> Result, ShellError> { - let tz: Tz = if let Some(polars_tz) = timezone { - polars_tz - .parse::() - .map_err(|_| ShellError::GenericError { - error: format!("Could not parse polars timezone: {polars_tz}"), - msg: "".to_string(), - span: Some(span), - help: None, - inner: vec![], - })? - } else { - Tz::UTC - }; - - Ok(tz.timestamp_nanos(nanos).fixed_offset()) -} - -fn time_from_midnight(nanos: i64, span: Span) -> Result { - let today = Utc::now().date_naive(); - NaiveTime::from_hms_opt(0, 0, 0) // midnight - .map(|time| time + Duration::nanoseconds(nanos)) // current time - .map(|time| today.and_time(time)) // current date and time - .and_then(|datetime| { - FixedOffset::east_opt(0) // utc - .map(|offset| { - DateTime::::from_naive_utc_and_offset(datetime, offset) - }) - }) - .map(|datetime| Value::date(datetime, span)) // current date and time - .ok_or(ShellError::CantConvert { - to_type: "datetime".to_string(), - from_type: "polars time".to_string(), - span, - help: Some("Could not convert polars time of {nanos} to datetime".to_string()), - }) -} - -#[cfg(test)] -mod tests { - use indexmap::indexmap; - use nu_protocol::record; - use polars::export::arrow::array::{BooleanArray, PrimitiveArray}; - use polars::prelude::Field; - use polars_io::prelude::StructArray; - - use super::*; - - #[test] - fn test_parsed_column_string_list() -> Result<(), Box> { - let values = vec![ - Value::list( - vec![Value::string("bar".to_string(), Span::test_data())], - Span::test_data(), - ), - Value::list( - vec![Value::string("baz".to_string(), Span::test_data())], - Span::test_data(), - ), - ]; - let column = Column { - name: "foo".to_string(), - values: values.clone(), - }; - let typed_column = TypedColumn { - column, - column_type: Some(DataType::List(Box::new(DataType::String))), - }; - - let column_map = indexmap!("foo".to_string() => typed_column); - let parsed_df = from_parsed_columns(column_map)?; - let parsed_columns = parsed_df.columns(Span::test_data())?; - assert_eq!(parsed_columns.len(), 1); - let column = parsed_columns - .first() - .expect("There should be a first value in columns"); - assert_eq!(column.name(), "foo"); - assert_eq!(column.values, values); - - Ok(()) - } - - #[test] - fn test_any_value_to_value() -> Result<(), Box> { - let span = Span::test_data(); - assert_eq!( - any_value_to_value(&AnyValue::Null, span)?, - Value::nothing(span) - ); - - let test_bool = true; - assert_eq!( - any_value_to_value(&AnyValue::Boolean(test_bool), span)?, - Value::bool(test_bool, span) - ); - - let test_str = "foo"; - assert_eq!( - any_value_to_value(&AnyValue::String(test_str), span)?, - Value::string(test_str.to_string(), span) - ); - assert_eq!( - any_value_to_value(&AnyValue::StringOwned(test_str.into()), span)?, - Value::string(test_str.to_owned(), span) - ); - - let tests_uint8 = 4; - assert_eq!( - any_value_to_value(&AnyValue::UInt8(tests_uint8), span)?, - Value::int(tests_uint8 as i64, span) - ); - - let tests_uint16 = 233; - assert_eq!( - any_value_to_value(&AnyValue::UInt16(tests_uint16), span)?, - Value::int(tests_uint16 as i64, span) - ); - - let tests_uint32 = 897688233; - assert_eq!( - any_value_to_value(&AnyValue::UInt32(tests_uint32), span)?, - Value::int(tests_uint32 as i64, span) - ); - - let tests_uint64 = 903225135897388233; - assert_eq!( - any_value_to_value(&AnyValue::UInt64(tests_uint64), span)?, - Value::int(tests_uint64 as i64, span) - ); - - let tests_float32 = 903225135897388233.3223353; - assert_eq!( - any_value_to_value(&AnyValue::Float32(tests_float32), span)?, - Value::float(tests_float32 as f64, span) - ); - - let tests_float64 = 9064251358973882322333.64233533232; - assert_eq!( - any_value_to_value(&AnyValue::Float64(tests_float64), span)?, - Value::float(tests_float64, span) - ); - - let test_days = 10_957; - let comparison_date = Utc - .with_ymd_and_hms(2000, 1, 1, 0, 0, 0) - .unwrap() - .fixed_offset(); - assert_eq!( - any_value_to_value(&AnyValue::Date(test_days), span)?, - Value::date(comparison_date, span) - ); - - let test_millis = 946_684_800_000; - assert_eq!( - any_value_to_value( - &AnyValue::Datetime(test_millis, TimeUnit::Milliseconds, &None), - span - )?, - Value::date(comparison_date, span) - ); - - let test_duration_millis = 99_999; - let test_duration_micros = 99_999_000; - let test_duration_nanos = 99_999_000_000; - assert_eq!( - any_value_to_value( - &AnyValue::Duration(test_duration_nanos, TimeUnit::Nanoseconds), - span - )?, - Value::duration(test_duration_nanos, span) - ); - assert_eq!( - any_value_to_value( - &AnyValue::Duration(test_duration_micros, TimeUnit::Microseconds), - span - )?, - Value::duration(test_duration_nanos, span) - ); - assert_eq!( - any_value_to_value( - &AnyValue::Duration(test_duration_millis, TimeUnit::Milliseconds), - span - )?, - Value::duration(test_duration_nanos, span) - ); - - let test_binary = b"sdf2332f32q3f3afwaf3232f32"; - assert_eq!( - any_value_to_value(&AnyValue::Binary(test_binary), span)?, - Value::binary(test_binary.to_vec(), span) - ); - assert_eq!( - any_value_to_value(&AnyValue::BinaryOwned(test_binary.to_vec()), span)?, - Value::binary(test_binary.to_vec(), span) - ); - - let test_time_nanos = 54_000_000_000_000; - let test_time = DateTime::::from_naive_utc_and_offset( - Utc::now() - .date_naive() - .and_time(NaiveTime::from_hms_opt(15, 00, 00).unwrap()), - FixedOffset::east_opt(0).unwrap(), - ); - assert_eq!( - any_value_to_value(&AnyValue::Time(test_time_nanos), span)?, - Value::date(test_time, span) - ); - - let test_list_series = Series::new("int series", &[1, 2, 3]); - let comparison_list_series = Value::list( - vec![ - Value::int(1, span), - Value::int(2, span), - Value::int(3, span), - ], - span, - ); - assert_eq!( - any_value_to_value(&AnyValue::List(test_list_series), span)?, - comparison_list_series - ); - - let field_value_0 = AnyValue::Int32(1); - let field_value_1 = AnyValue::Boolean(true); - let values = vec![field_value_0, field_value_1]; - let field_name_0 = "num_field"; - let field_name_1 = "bool_field"; - let fields = vec![ - Field::new(field_name_0, DataType::Int32), - Field::new(field_name_1, DataType::Boolean), - ]; - let test_owned_struct = AnyValue::StructOwned(Box::new((values, fields.clone()))); - let comparison_owned_record = Value::test_record(record!( - field_name_0 => Value::int(1, span), - field_name_1 => Value::bool(true, span), - )); - assert_eq!( - any_value_to_value(&test_owned_struct, span)?, - comparison_owned_record.clone() - ); - - let test_int_arr = PrimitiveArray::from([Some(1_i32)]); - let test_bool_arr = BooleanArray::from([Some(true)]); - let test_struct_arr = StructArray::new( - DataType::Struct(fields.clone()).to_arrow(true), - vec![Box::new(test_int_arr), Box::new(test_bool_arr)], - None, - ); - assert_eq!( - any_value_to_value( - &AnyValue::Struct(0, &test_struct_arr, fields.as_slice()), - span - )?, - comparison_owned_record - ); - - Ok(()) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/custom_value.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/custom_value.rs deleted file mode 100644 index da8b27398b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/custom_value.rs +++ /dev/null @@ -1,79 +0,0 @@ -use super::NuDataFrame; -use nu_protocol::{ast::Operator, CustomValue, ShellError, Span, Value}; - -// CustomValue implementation for NuDataFrame -impl CustomValue for NuDataFrame { - fn typetag_name(&self) -> &'static str { - "dataframe" - } - - fn typetag_deserialize(&self) { - unimplemented!("typetag_deserialize") - } - - fn clone_value(&self, span: nu_protocol::Span) -> Value { - let cloned = NuDataFrame { - df: self.df.clone(), - from_lazy: false, - }; - - Value::custom(Box::new(cloned), span) - } - - fn type_name(&self) -> String { - self.typetag_name().to_string() - } - - fn to_base_value(&self, span: Span) -> Result { - let vals = self.print(span)?; - - Ok(Value::list(vals, span)) - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - fn follow_path_int( - &self, - _self_span: Span, - count: usize, - path_span: Span, - ) -> Result { - self.get_value(count, path_span) - } - - fn follow_path_string( - &self, - _self_span: Span, - column_name: String, - path_span: Span, - ) -> Result { - let column = self.column(&column_name, path_span)?; - Ok(column.into_value(path_span)) - } - - fn partial_cmp(&self, other: &Value) -> Option { - match other { - Value::Custom { val, .. } => val - .as_any() - .downcast_ref::() - .and_then(|other| self.is_equal(other)), - _ => None, - } - } - - fn operation( - &self, - lhs_span: Span, - operator: Operator, - op: Span, - right: &Value, - ) -> Result { - self.compute_with_value(lhs_span, operator, op, right) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs deleted file mode 100644 index 967e03580f..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs +++ /dev/null @@ -1,580 +0,0 @@ -mod between_values; -mod conversion; -mod custom_value; -mod operations; - -pub use conversion::{Column, ColumnMap}; -pub use operations::Axis; - -use super::{nu_schema::NuSchema, utils::DEFAULT_ROWS, NuLazyFrame}; -use indexmap::IndexMap; -use nu_protocol::{did_you_mean, PipelineData, Record, ShellError, Span, Value}; -use polars::{ - chunked_array::ops::SortMultipleOptions, - prelude::{DataFrame, DataType, IntoLazy, LazyFrame, PolarsObject, Series}, -}; -use polars_plan::prelude::{lit, Expr, Null}; -use polars_utils::total_ord::{TotalEq, TotalHash}; -use serde::{Deserialize, Serialize}; -use std::{ - cmp::Ordering, - collections::HashSet, - fmt::Display, - hash::{Hash, Hasher}, -}; - -// DataFrameValue is an encapsulation of Nushell Value that can be used -// to define the PolarsObject Trait. The polars object trait allows to -// create dataframes with mixed datatypes -#[derive(Clone, Debug)] -pub struct DataFrameValue(Value); - -impl DataFrameValue { - fn new(value: Value) -> Self { - Self(value) - } - - fn get_value(&self) -> Value { - self.0.clone() - } -} - -impl TotalHash for DataFrameValue { - fn tot_hash(&self, state: &mut H) - where - H: Hasher, - { - (*self).hash(state) - } -} - -impl Display for DataFrameValue { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0.get_type()) - } -} - -impl Default for DataFrameValue { - fn default() -> Self { - Self(Value::nothing(Span::unknown())) - } -} - -impl PartialEq for DataFrameValue { - fn eq(&self, other: &Self) -> bool { - self.0.partial_cmp(&other.0).map_or(false, Ordering::is_eq) - } -} -impl Eq for DataFrameValue {} - -impl Hash for DataFrameValue { - fn hash(&self, state: &mut H) { - match &self.0 { - Value::Nothing { .. } => 0.hash(state), - Value::Int { val, .. } => val.hash(state), - Value::String { val, .. } => val.hash(state), - // TODO. Define hash for the rest of types - _ => {} - } - } -} - -impl TotalEq for DataFrameValue { - fn tot_eq(&self, other: &Self) -> bool { - self == other - } -} - -impl PolarsObject for DataFrameValue { - fn type_name() -> &'static str { - "object" - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct NuDataFrame { - pub df: DataFrame, - pub from_lazy: bool, -} - -impl AsRef for NuDataFrame { - fn as_ref(&self) -> &polars::prelude::DataFrame { - &self.df - } -} - -impl AsMut for NuDataFrame { - fn as_mut(&mut self) -> &mut polars::prelude::DataFrame { - &mut self.df - } -} - -impl From for NuDataFrame { - fn from(df: DataFrame) -> Self { - Self { - df, - from_lazy: false, - } - } -} - -impl NuDataFrame { - pub fn new(from_lazy: bool, df: DataFrame) -> Self { - Self { df, from_lazy } - } - - pub fn lazy(&self) -> LazyFrame { - self.df.clone().lazy() - } - - fn default_value(span: Span) -> Value { - let dataframe = DataFrame::default(); - NuDataFrame::dataframe_into_value(dataframe, span) - } - - pub fn dataframe_into_value(dataframe: DataFrame, span: Span) -> Value { - Value::custom(Box::new(Self::new(false, dataframe)), span) - } - - pub fn into_value(self, span: Span) -> Value { - if self.from_lazy { - let lazy = NuLazyFrame::from_dataframe(self); - Value::custom(Box::new(lazy), span) - } else { - Value::custom(Box::new(self), span) - } - } - - pub fn series_to_value(series: Series, span: Span) -> Result { - match DataFrame::new(vec![series]) { - Ok(dataframe) => Ok(NuDataFrame::dataframe_into_value(dataframe, span)), - Err(e) => Err(ShellError::GenericError { - error: "Error creating dataframe".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - - pub fn try_from_iter(iter: T, maybe_schema: Option) -> Result - where - T: Iterator, - { - // Dictionary to store the columnar data extracted from - // the input. During the iteration we check if the values - // have different type - let mut column_values: ColumnMap = IndexMap::new(); - - for value in iter { - match value { - Value::Custom { .. } => return Self::try_from_value(value), - Value::List { vals, .. } => { - let record = vals - .into_iter() - .enumerate() - .map(|(i, val)| (format!("{i}"), val)) - .collect(); - - conversion::insert_record(&mut column_values, record, &maybe_schema)? - } - Value::Record { val: record, .. } => conversion::insert_record( - &mut column_values, - record.into_owned(), - &maybe_schema, - )?, - _ => { - let key = "0".to_string(); - conversion::insert_value(value, key, &mut column_values, &maybe_schema)? - } - } - } - - let df = conversion::from_parsed_columns(column_values)?; - add_missing_columns(df, &maybe_schema, Span::unknown()) - } - - pub fn try_from_series(columns: Vec, span: Span) -> Result { - let dataframe = DataFrame::new(columns).map_err(|e| ShellError::GenericError { - error: "Error creating dataframe".into(), - msg: format!("Unable to create DataFrame: {e}"), - span: Some(span), - help: None, - inner: vec![], - })?; - - Ok(Self::new(false, dataframe)) - } - - pub fn try_from_columns( - columns: Vec, - maybe_schema: Option, - ) -> Result { - let mut column_values: ColumnMap = IndexMap::new(); - - for column in columns { - let name = column.name().to_string(); - for value in column { - conversion::insert_value(value, name.clone(), &mut column_values, &maybe_schema)?; - } - } - - let df = conversion::from_parsed_columns(column_values)?; - add_missing_columns(df, &maybe_schema, Span::unknown()) - } - - pub fn fill_list_nan(list: Vec, list_span: Span, fill: Value) -> Value { - let newlist = list - .into_iter() - .map(|value| { - let span = value.span(); - match value { - Value::Float { val, .. } => { - if val.is_nan() { - fill.clone() - } else { - value - } - } - Value::List { vals, .. } => Self::fill_list_nan(vals, span, fill.clone()), - _ => value, - } - }) - .collect::>(); - Value::list(newlist, list_span) - } - - pub fn columns(&self, span: Span) -> Result, ShellError> { - let height = self.df.height(); - self.df - .get_columns() - .iter() - .map(|col| conversion::create_column(col, 0, height, span)) - .collect::, ShellError>>() - } - - pub fn try_from_value(value: Value) -> Result { - if Self::can_downcast(&value) { - Ok(Self::get_df(value)?) - } else if NuLazyFrame::can_downcast(&value) { - let span = value.span(); - let lazy = NuLazyFrame::try_from_value(value)?; - let df = lazy.collect(span)?; - Ok(df) - } else { - Err(ShellError::CantConvert { - to_type: "lazy or eager dataframe".into(), - from_type: value.get_type().to_string(), - span: value.span(), - help: None, - }) - } - } - - pub fn get_df(value: Value) -> Result { - let span = value.span(); - match value { - Value::Custom { val, .. } => match val.as_any().downcast_ref::() { - Some(df) => Ok(NuDataFrame { - df: df.df.clone(), - from_lazy: false, - }), - None => Err(ShellError::CantConvert { - to_type: "dataframe".into(), - from_type: "non-dataframe".into(), - span, - help: None, - }), - }, - x => Err(ShellError::CantConvert { - to_type: "dataframe".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }), - } - } - - pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span)?; - Self::try_from_value(value) - } - - pub fn can_downcast(value: &Value) -> bool { - if let Value::Custom { val, .. } = value { - val.as_any().downcast_ref::().is_some() - } else { - false - } - } - - pub fn column(&self, column: &str, span: Span) -> Result { - let s = self.df.column(column).map_err(|_| { - let possibilities = self - .df - .get_column_names() - .iter() - .map(|name| name.to_string()) - .collect::>(); - - let option = did_you_mean(&possibilities, column).unwrap_or_else(|| column.to_string()); - ShellError::DidYouMean { - suggestion: option, - span, - } - })?; - - let df = DataFrame::new(vec![s.clone()]).map_err(|e| ShellError::GenericError { - error: "Error creating dataframe".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - Ok(Self { - df, - from_lazy: false, - }) - } - - pub fn is_series(&self) -> bool { - self.df.width() == 1 - } - - pub fn as_series(&self, span: Span) -> Result { - if !self.is_series() { - return Err(ShellError::GenericError { - error: "Error using as series".into(), - msg: "dataframe has more than one column".into(), - span: Some(span), - help: None, - inner: vec![], - }); - } - - let series = self - .df - .get_columns() - .first() - .expect("We have already checked that the width is 1"); - - Ok(series.clone()) - } - - pub fn get_value(&self, row: usize, span: Span) -> Result { - let series = self.as_series(span)?; - let column = conversion::create_column(&series, row, row + 1, span)?; - - if column.len() == 0 { - Err(ShellError::AccessEmptyContent { span }) - } else { - let value = column - .into_iter() - .next() - .expect("already checked there is a value"); - Ok(value) - } - } - - // Print is made out a head and if the dataframe is too large, then a tail - pub fn print(&self, span: Span) -> Result, ShellError> { - let df = &self.df; - let size: usize = 20; - - if df.height() > size { - let sample_size = size / 2; - let mut values = self.head(Some(sample_size), span)?; - conversion::add_separator(&mut values, df, span); - let remaining = df.height() - sample_size; - let tail_size = remaining.min(sample_size); - let mut tail_values = self.tail(Some(tail_size), span)?; - values.append(&mut tail_values); - - Ok(values) - } else { - Ok(self.head(Some(size), span)?) - } - } - - pub fn height(&self) -> usize { - self.df.height() - } - - pub fn head(&self, rows: Option, span: Span) -> Result, ShellError> { - let to_row = rows.unwrap_or(5); - let values = self.to_rows(0, to_row, span)?; - - Ok(values) - } - - pub fn tail(&self, rows: Option, span: Span) -> Result, ShellError> { - let df = &self.df; - let to_row = df.height(); - let size = rows.unwrap_or(DEFAULT_ROWS); - let from_row = to_row.saturating_sub(size); - - let values = self.to_rows(from_row, to_row, span)?; - - Ok(values) - } - - pub fn to_rows( - &self, - from_row: usize, - to_row: usize, - span: Span, - ) -> Result, ShellError> { - let df = &self.df; - let upper_row = to_row.min(df.height()); - - let mut size: usize = 0; - let columns = self - .df - .get_columns() - .iter() - .map( - |col| match conversion::create_column(col, from_row, upper_row, span) { - Ok(col) => { - size = col.len(); - Ok(col) - } - Err(e) => Err(e), - }, - ) - .collect::, ShellError>>()?; - - let mut iterators = columns - .into_iter() - .map(|col| (col.name().to_string(), col.into_iter())) - .collect::)>>(); - - let values = (0..size) - .map(|i| { - let mut record = Record::new(); - - record.push("index", Value::int((i + from_row) as i64, span)); - - for (name, col) in &mut iterators { - record.push(name.clone(), col.next().unwrap_or(Value::nothing(span))); - } - - Value::record(record, span) - }) - .collect::>(); - - Ok(values) - } - - // Dataframes are considered equal if they have the same shape, column name and values - pub fn is_equal(&self, other: &Self) -> Option { - if self.as_ref().width() == 0 { - // checking for empty dataframe - return None; - } - - if self.as_ref().get_column_names() != other.as_ref().get_column_names() { - // checking both dataframes share the same names - return None; - } - - if self.as_ref().height() != other.as_ref().height() { - // checking both dataframes have the same row size - return None; - } - - // sorting dataframe by the first column - let column_names = self.as_ref().get_column_names(); - let first_col = column_names - .first() - .expect("already checked that dataframe is different than 0"); - - // if unable to sort, then unable to compare - let lhs = match self - .as_ref() - .sort(vec![*first_col], SortMultipleOptions::default()) - { - Ok(df) => df, - Err(_) => return None, - }; - - let rhs = match other - .as_ref() - .sort(vec![*first_col], SortMultipleOptions::default()) - { - Ok(df) => df, - Err(_) => return None, - }; - - for name in self.as_ref().get_column_names() { - let self_series = lhs.column(name).expect("name from dataframe names"); - - let other_series = rhs - .column(name) - .expect("already checked that name in other"); - - let self_series = match self_series.dtype() { - // Casting needed to compare other numeric types with nushell numeric type. - // In nushell we only have i64 integer numeric types and any array created - // with nushell untagged primitives will be of type i64 - DataType::UInt32 | DataType::Int32 => match self_series.cast(&DataType::Int64) { - Ok(series) => series, - Err(_) => return None, - }, - _ => self_series.clone(), - }; - - if !self_series.equals(other_series) { - return None; - } - } - - Some(Ordering::Equal) - } - - pub fn schema(&self) -> NuSchema { - NuSchema::new(self.df.schema()) - } -} - -fn add_missing_columns( - df: NuDataFrame, - maybe_schema: &Option, - span: Span, -) -> Result { - // If there are fields that are in the schema, but not in the dataframe - // add them to the dataframe. - if let Some(schema) = maybe_schema { - let fields = df.df.fields(); - let df_field_names: HashSet<&str> = fields.iter().map(|f| f.name().as_str()).collect(); - - let missing: Vec<(&str, &DataType)> = schema - .schema - .iter() - .filter_map(|(name, dtype)| { - let name = name.as_str(); - if !df_field_names.contains(name) { - Some((name, dtype)) - } else { - None - } - }) - .collect(); - - let missing_exprs: Vec = missing - .iter() - .map(|(name, dtype)| lit(Null {}).cast((*dtype).to_owned()).alias(name)) - .collect(); - - let df = if !missing.is_empty() { - let with_columns = df.lazy().with_columns(missing_exprs); - NuLazyFrame::new(true, with_columns).collect(span)? - } else { - df - }; - Ok(df) - } else { - Ok(df) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/operations.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/operations.rs deleted file mode 100644 index ff2f7b7604..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/operations.rs +++ /dev/null @@ -1,206 +0,0 @@ -use super::{ - between_values::{between_dataframes, compute_between_series, compute_series_single_value}, - NuDataFrame, -}; -use nu_protocol::{ast::Operator, ShellError, Span, Spanned, Value}; -use polars::prelude::{DataFrame, Series}; - -pub enum Axis { - Row, - Column, -} - -impl NuDataFrame { - pub fn compute_with_value( - &self, - lhs_span: Span, - operator: Operator, - op_span: Span, - right: &Value, - ) -> Result { - let rhs_span = right.span(); - match right { - Value::Custom { val: rhs, .. } => { - let rhs = rhs.as_any().downcast_ref::().ok_or_else(|| { - ShellError::DowncastNotPossible { - msg: "Unable to create dataframe".to_string(), - span: rhs_span, - } - })?; - - match (self.is_series(), rhs.is_series()) { - (true, true) => { - let lhs = &self - .as_series(lhs_span) - .expect("Already checked that is a series"); - let rhs = &rhs - .as_series(rhs_span) - .expect("Already checked that is a series"); - - if lhs.dtype() != rhs.dtype() { - return Err(ShellError::IncompatibleParameters { - left_message: format!("datatype {}", lhs.dtype()), - left_span: lhs_span, - right_message: format!("datatype {}", lhs.dtype()), - right_span: rhs_span, - }); - } - - if lhs.len() != rhs.len() { - return Err(ShellError::IncompatibleParameters { - left_message: format!("len {}", lhs.len()), - left_span: lhs_span, - right_message: format!("len {}", rhs.len()), - right_span: rhs_span, - }); - } - - let op = Spanned { - item: operator, - span: op_span, - }; - - compute_between_series( - op, - &NuDataFrame::default_value(lhs_span), - lhs, - right, - rhs, - ) - } - _ => { - if self.df.height() != rhs.df.height() { - return Err(ShellError::IncompatibleParameters { - left_message: format!("rows {}", self.df.height()), - left_span: lhs_span, - right_message: format!("rows {}", rhs.df.height()), - right_span: rhs_span, - }); - } - - let op = Spanned { - item: operator, - span: op_span, - }; - - between_dataframes( - op, - &NuDataFrame::default_value(lhs_span), - self, - right, - rhs, - ) - } - } - } - _ => { - let op = Spanned { - item: operator, - span: op_span, - }; - - compute_series_single_value(op, &NuDataFrame::default_value(lhs_span), self, right) - } - } - } - - pub fn append_df( - &self, - other: &NuDataFrame, - axis: Axis, - span: Span, - ) -> Result { - match axis { - Axis::Row => { - let mut columns: Vec<&str> = Vec::new(); - - let new_cols = self - .df - .get_columns() - .iter() - .chain(other.df.get_columns()) - .map(|s| { - let name = if columns.contains(&s.name()) { - format!("{}_{}", s.name(), "x") - } else { - columns.push(s.name()); - s.name().to_string() - }; - - let mut series = s.clone(); - series.rename(&name); - series - }) - .collect::>(); - - let df_new = DataFrame::new(new_cols).map_err(|e| ShellError::GenericError { - error: "Error creating dataframe".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - Ok(NuDataFrame::new(false, df_new)) - } - Axis::Column => { - if self.df.width() != other.df.width() { - return Err(ShellError::IncompatibleParametersSingle { - msg: "Dataframes with different number of columns".into(), - span, - }); - } - - if !self - .df - .get_column_names() - .iter() - .all(|col| other.df.get_column_names().contains(col)) - { - return Err(ShellError::IncompatibleParametersSingle { - msg: "Dataframes with different columns names".into(), - span, - }); - } - - let new_cols = self - .df - .get_columns() - .iter() - .map(|s| { - let other_col = other - .df - .column(s.name()) - .expect("Already checked that dataframes have same columns"); - - let mut tmp = s.clone(); - let res = tmp.append(other_col); - - match res { - Ok(s) => Ok(s.clone()), - Err(e) => Err({ - ShellError::GenericError { - error: "Error appending dataframe".into(), - msg: format!("Unable to append: {e}"), - span: Some(span), - help: None, - inner: vec![], - } - }), - } - }) - .collect::, ShellError>>()?; - - let df_new = DataFrame::new(new_cols).map_err(|e| ShellError::GenericError { - error: "Error appending dataframe".into(), - msg: format!("Unable to append dataframes: {e}"), - span: Some(span), - help: None, - inner: vec![], - })?; - - Ok(NuDataFrame::new(false, df_new)) - } - } - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/custom_value.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/custom_value.rs deleted file mode 100644 index 7a7f59e648..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/custom_value.rs +++ /dev/null @@ -1,147 +0,0 @@ -use super::NuExpression; -use nu_protocol::{ - ast::{Comparison, Math, Operator}, - CustomValue, ShellError, Span, Type, Value, -}; -use polars::prelude::Expr; -use std::ops::{Add, Div, Mul, Rem, Sub}; - -// CustomValue implementation for NuDataFrame -impl CustomValue for NuExpression { - fn typetag_name(&self) -> &'static str { - "expression" - } - - fn typetag_deserialize(&self) { - unimplemented!("typetag_deserialize") - } - - fn clone_value(&self, span: nu_protocol::Span) -> Value { - let cloned = NuExpression(self.0.clone()); - - Value::custom(Box::new(cloned), span) - } - - fn type_name(&self) -> String { - self.typetag_name().to_string() - } - - fn to_base_value(&self, span: Span) -> Result { - self.to_value(span) - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - fn operation( - &self, - lhs_span: Span, - operator: Operator, - op: Span, - right: &Value, - ) -> Result { - compute_with_value(self, lhs_span, operator, op, right) - } -} - -fn compute_with_value( - left: &NuExpression, - lhs_span: Span, - operator: Operator, - op: Span, - right: &Value, -) -> Result { - let rhs_span = right.span(); - match right { - Value::Custom { val: rhs, .. } => { - let rhs = rhs.as_any().downcast_ref::().ok_or_else(|| { - ShellError::DowncastNotPossible { - msg: "Unable to create expression".into(), - span: rhs_span, - } - })?; - - match rhs.as_ref() { - polars::prelude::Expr::Literal(..) => { - with_operator(operator, left, rhs, lhs_span, right.span(), op) - } - _ => Err(ShellError::TypeMismatch { - err_message: "Only literal expressions or number".into(), - span: right.span(), - }), - } - } - _ => { - let rhs = NuExpression::try_from_value(right.clone())?; - with_operator(operator, left, &rhs, lhs_span, right.span(), op) - } - } -} - -fn with_operator( - operator: Operator, - left: &NuExpression, - right: &NuExpression, - lhs_span: Span, - rhs_span: Span, - op_span: Span, -) -> Result { - match operator { - Operator::Math(Math::Plus) => apply_arithmetic(left, right, lhs_span, Add::add), - Operator::Math(Math::Minus) => apply_arithmetic(left, right, lhs_span, Sub::sub), - Operator::Math(Math::Multiply) => apply_arithmetic(left, right, lhs_span, Mul::mul), - Operator::Math(Math::Divide) => apply_arithmetic(left, right, lhs_span, Div::div), - Operator::Math(Math::Modulo) => apply_arithmetic(left, right, lhs_span, Rem::rem), - Operator::Math(Math::FloorDivision) => apply_arithmetic(left, right, lhs_span, Div::div), - Operator::Comparison(Comparison::Equal) => Ok(left - .clone() - .apply_with_expr(right.clone(), Expr::eq) - .into_value(lhs_span)), - Operator::Comparison(Comparison::NotEqual) => Ok(left - .clone() - .apply_with_expr(right.clone(), Expr::neq) - .into_value(lhs_span)), - Operator::Comparison(Comparison::GreaterThan) => Ok(left - .clone() - .apply_with_expr(right.clone(), Expr::gt) - .into_value(lhs_span)), - Operator::Comparison(Comparison::GreaterThanOrEqual) => Ok(left - .clone() - .apply_with_expr(right.clone(), Expr::gt_eq) - .into_value(lhs_span)), - Operator::Comparison(Comparison::LessThan) => Ok(left - .clone() - .apply_with_expr(right.clone(), Expr::lt) - .into_value(lhs_span)), - Operator::Comparison(Comparison::LessThanOrEqual) => Ok(left - .clone() - .apply_with_expr(right.clone(), Expr::lt_eq) - .into_value(lhs_span)), - _ => Err(ShellError::OperatorMismatch { - op_span, - lhs_ty: Type::Custom(left.typetag_name().into()).to_string(), - lhs_span, - rhs_ty: Type::Custom(right.typetag_name().into()).to_string(), - rhs_span, - }), - } -} - -fn apply_arithmetic( - left: &NuExpression, - right: &NuExpression, - span: Span, - f: F, -) -> Result -where - F: Fn(Expr, Expr) -> Expr, -{ - let expr: NuExpression = f(left.as_ref().clone(), right.as_ref().clone()).into(); - - Ok(expr.into_value(span)) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs deleted file mode 100644 index cee31d7b53..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs +++ /dev/null @@ -1,443 +0,0 @@ -mod custom_value; - -use nu_protocol::{record, PipelineData, ShellError, Span, Value}; -use polars::prelude::{col, AggExpr, Expr, Literal}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -// Polars Expression wrapper for Nushell operations -// Object is behind and Option to allow easy implementation of -// the Deserialize trait -#[derive(Default, Clone, Debug)] -pub struct NuExpression(Option); - -// Mocked serialization of the LazyFrame object -impl Serialize for NuExpression { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_none() - } -} - -// Mocked deserialization of the LazyFrame object -impl<'de> Deserialize<'de> for NuExpression { - fn deserialize(_deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Ok(NuExpression::default()) - } -} - -// Referenced access to the real LazyFrame -impl AsRef for NuExpression { - fn as_ref(&self) -> &polars::prelude::Expr { - // The only case when there cannot be an expr is if it is created - // using the default function or if created by deserializing something - self.0.as_ref().expect("there should always be a frame") - } -} - -impl AsMut for NuExpression { - fn as_mut(&mut self) -> &mut polars::prelude::Expr { - // The only case when there cannot be an expr is if it is created - // using the default function or if created by deserializing something - self.0.as_mut().expect("there should always be a frame") - } -} - -impl From for NuExpression { - fn from(expr: Expr) -> Self { - Self(Some(expr)) - } -} - -impl NuExpression { - pub fn into_value(self, span: Span) -> Value { - Value::custom(Box::new(self), span) - } - - pub fn try_from_value(value: Value) -> Result { - let span = value.span(); - match value { - Value::Custom { val, .. } => match val.as_any().downcast_ref::() { - Some(expr) => Ok(NuExpression(expr.0.clone())), - None => Err(ShellError::CantConvert { - to_type: "lazy expression".into(), - from_type: "non-dataframe".into(), - span, - help: None, - }), - }, - Value::String { val, .. } => Ok(val.lit().into()), - Value::Int { val, .. } => Ok(val.lit().into()), - Value::Bool { val, .. } => Ok(val.lit().into()), - Value::Float { val, .. } => Ok(val.lit().into()), - x => Err(ShellError::CantConvert { - to_type: "lazy expression".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }), - } - } - - pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span)?; - Self::try_from_value(value) - } - - pub fn can_downcast(value: &Value) -> bool { - match value { - Value::Custom { val, .. } => val.as_any().downcast_ref::().is_some(), - Value::List { vals, .. } => vals.iter().all(Self::can_downcast), - Value::String { .. } | Value::Int { .. } | Value::Bool { .. } | Value::Float { .. } => { - true - } - _ => false, - } - } - - pub fn into_polars(self) -> Expr { - self.0.expect("Expression cannot be none to convert") - } - - pub fn apply_with_expr(self, other: NuExpression, f: F) -> Self - where - F: Fn(Expr, Expr) -> Expr, - { - let expr = self.0.expect("Lazy expression must not be empty to apply"); - let other = other.0.expect("Lazy expression must not be empty to apply"); - - f(expr, other).into() - } - - pub fn to_value(&self, span: Span) -> Result { - expr_to_value(self.as_ref(), span) - } - - // Convenient function to extract multiple Expr that could be inside a nushell Value - pub fn extract_exprs(value: Value) -> Result, ShellError> { - ExtractedExpr::extract_exprs(value).map(ExtractedExpr::into_exprs) - } -} - -#[derive(Debug)] -// Enum to represent the parsing of the expressions from Value -enum ExtractedExpr { - Single(Expr), - List(Vec), -} - -impl ExtractedExpr { - fn into_exprs(self) -> Vec { - match self { - Self::Single(expr) => vec![expr], - Self::List(expressions) => expressions - .into_iter() - .flat_map(ExtractedExpr::into_exprs) - .collect(), - } - } - - fn extract_exprs(value: Value) -> Result { - match value { - Value::String { val, .. } => Ok(ExtractedExpr::Single(col(val.as_str()))), - Value::Custom { .. } => NuExpression::try_from_value(value) - .map(NuExpression::into_polars) - .map(ExtractedExpr::Single), - Value::List { vals, .. } => vals - .into_iter() - .map(Self::extract_exprs) - .collect::, ShellError>>() - .map(ExtractedExpr::List), - x => Err(ShellError::CantConvert { - to_type: "expression".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }), - } - } -} - -pub fn expr_to_value(expr: &Expr, span: Span) -> Result { - match expr { - Expr::Alias(expr, alias) => Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "alias" => Value::string(alias.as_ref(), span), - }, - span, - )), - Expr::Column(name) => Ok(Value::record( - record! { - "expr" => Value::string("column", span), - "value" => Value::string(name.to_string(), span), - }, - span, - )), - Expr::Columns(columns) => { - let value = columns.iter().map(|col| Value::string(col, span)).collect(); - Ok(Value::record( - record! { - "expr" => Value::string("columns", span), - "value" => Value::list(value, span), - }, - span, - )) - } - Expr::Literal(literal) => Ok(Value::record( - record! { - "expr" => Value::string("literal", span), - "value" => Value::string(format!("{literal:?}"), span), - }, - span, - )), - Expr::BinaryExpr { left, op, right } => Ok(Value::record( - record! { - "left" => expr_to_value(left, span)?, - "op" => Value::string(format!("{op:?}"), span), - "right" => expr_to_value(right, span)?, - }, - span, - )), - Expr::Ternary { - predicate, - truthy, - falsy, - } => Ok(Value::record( - record! { - "predicate" => expr_to_value(predicate.as_ref(), span)?, - "truthy" => expr_to_value(truthy.as_ref(), span)?, - "falsy" => expr_to_value(falsy.as_ref(), span)?, - }, - span, - )), - Expr::Agg(agg_expr) => { - let value = match agg_expr { - AggExpr::Min { input: expr, .. } - | AggExpr::Max { input: expr, .. } - | AggExpr::Median(expr) - | AggExpr::NUnique(expr) - | AggExpr::First(expr) - | AggExpr::Last(expr) - | AggExpr::Mean(expr) - | AggExpr::Implode(expr) - | AggExpr::Count(expr, _) - | AggExpr::Sum(expr) - | AggExpr::AggGroups(expr) - | AggExpr::Std(expr, _) - | AggExpr::Var(expr, _) => expr_to_value(expr.as_ref(), span), - AggExpr::Quantile { - expr, - quantile, - interpol, - } => Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "quantile" => expr_to_value(quantile.as_ref(), span)?, - "interpol" => Value::string(format!("{interpol:?}"), span), - }, - span, - )), - }; - - Ok(Value::record( - record! { - "expr" => Value::string("agg", span), - "value" => value?, - }, - span, - )) - } - Expr::Len => Ok(Value::record( - record! { "expr" => Value::string("count", span) }, - span, - )), - Expr::Wildcard => Ok(Value::record( - record! { "expr" => Value::string("wildcard", span) }, - span, - )), - Expr::Explode(expr) => Ok(Value::record( - record! { "expr" => expr_to_value(expr.as_ref(), span)? }, - span, - )), - Expr::KeepName(expr) => Ok(Value::record( - record! { "expr" => expr_to_value(expr.as_ref(), span)? }, - span, - )), - Expr::Nth(i) => Ok(Value::record( - record! { "expr" => Value::int(*i, span) }, - span, - )), - Expr::DtypeColumn(dtypes) => { - let vals = dtypes - .iter() - .map(|d| Value::string(format!("{d}"), span)) - .collect(); - - Ok(Value::list(vals, span)) - } - Expr::Sort { expr, options } => Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "options" => Value::string(format!("{options:?}"), span), - }, - span, - )), - Expr::Cast { - expr, - data_type, - strict, - } => Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "dtype" => Value::string(format!("{data_type:?}"), span), - "strict" => Value::bool(*strict, span), - }, - span, - )), - Expr::Gather { - expr, - idx, - returns_scalar: _, - } => Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "idx" => expr_to_value(idx.as_ref(), span)?, - }, - span, - )), - Expr::SortBy { - expr, - by, - sort_options, - } => { - let by: Result, ShellError> = - by.iter().map(|b| expr_to_value(b, span)).collect(); - let descending: Vec = sort_options - .descending - .iter() - .map(|r| Value::bool(*r, span)) - .collect(); - - Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "by" => Value::list(by?, span), - "descending" => Value::list(descending, span), - }, - span, - )) - } - Expr::Filter { input, by } => Ok(Value::record( - record! { - "input" => expr_to_value(input.as_ref(), span)?, - "by" => expr_to_value(by.as_ref(), span)?, - }, - span, - )), - Expr::Slice { - input, - offset, - length, - } => Ok(Value::record( - record! { - "input" => expr_to_value(input.as_ref(), span)?, - "offset" => expr_to_value(offset.as_ref(), span)?, - "length" => expr_to_value(length.as_ref(), span)?, - }, - span, - )), - Expr::Exclude(expr, excluded) => { - let excluded = excluded - .iter() - .map(|e| Value::string(format!("{e:?}"), span)) - .collect(); - - Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "excluded" => Value::list(excluded, span), - }, - span, - )) - } - Expr::RenameAlias { expr, function } => Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "function" => Value::string(format!("{function:?}"), span), - }, - span, - )), - Expr::AnonymousFunction { - input, - function, - output_type, - options, - } => { - let input: Result, ShellError> = - input.iter().map(|e| expr_to_value(e, span)).collect(); - Ok(Value::record( - record! { - "input" => Value::list(input?, span), - "function" => Value::string(format!("{function:?}"), span), - "output_type" => Value::string(format!("{output_type:?}"), span), - "options" => Value::string(format!("{options:?}"), span), - }, - span, - )) - } - Expr::Function { - input, - function, - options, - } => { - let input: Result, ShellError> = - input.iter().map(|e| expr_to_value(e, span)).collect(); - Ok(Value::record( - record! { - "input" => Value::list(input?, span), - "function" => Value::string(format!("{function:?}"), span), - "options" => Value::string(format!("{options:?}"), span), - }, - span, - )) - } - Expr::Window { - function, - partition_by, - options, - } => { - let partition_by: Result, ShellError> = partition_by - .iter() - .map(|e| expr_to_value(e, span)) - .collect(); - - Ok(Value::record( - record! { - "function" => expr_to_value(function, span)?, - "partition_by" => Value::list(partition_by?, span), - "options" => Value::string(format!("{options:?}"), span), - }, - span, - )) - } - Expr::SubPlan(_, _) => Err(ShellError::UnsupportedInput { - msg: "Expressions of type SubPlan are not yet supported".to_string(), - input: format!("Expression is {expr:?}"), - msg_span: span, - input_span: Span::unknown(), - }), - // the parameter polars_plan::dsl::selector::Selector is not publicly exposed. - // I am not sure what we can meaningfully do with this at this time. - Expr::Selector(_) => Err(ShellError::UnsupportedInput { - msg: "Expressions of type Selector to Nu Values is not yet supported".to_string(), - input: format!("Expression is {expr:?}"), - msg_span: span, - input_span: Span::unknown(), - }), - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/custom_value.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/custom_value.rs deleted file mode 100644 index f747ae4d18..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/custom_value.rs +++ /dev/null @@ -1,50 +0,0 @@ -use super::NuLazyFrame; -use nu_protocol::{record, CustomValue, ShellError, Span, Value}; - -// CustomValue implementation for NuDataFrame -impl CustomValue for NuLazyFrame { - fn typetag_name(&self) -> &'static str { - "lazyframe" - } - - fn typetag_deserialize(&self) { - unimplemented!("typetag_deserialize") - } - - fn clone_value(&self, span: nu_protocol::Span) -> Value { - let cloned = NuLazyFrame { - lazy: self.lazy.clone(), - from_eager: self.from_eager, - schema: self.schema.clone(), - }; - - Value::custom(Box::new(cloned), span) - } - - fn type_name(&self) -> String { - self.typetag_name().to_string() - } - - fn to_base_value(&self, span: Span) -> Result { - let optimized_plan = self - .as_ref() - .describe_optimized_plan() - .unwrap_or_else(|_| "".to_string()); - - Ok(Value::record( - record! { - "plan" => Value::string(self.as_ref().describe_plan(), span), - "optimized_plan" => Value::string(optimized_plan, span), - }, - span, - )) - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs deleted file mode 100644 index 355516d340..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs +++ /dev/null @@ -1,188 +0,0 @@ -mod custom_value; - -use super::{NuDataFrame, NuExpression}; -use core::fmt; -use nu_protocol::{PipelineData, ShellError, Span, Value}; -use polars::prelude::{Expr, IntoLazy, LazyFrame, Schema}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -// Lazyframe wrapper for Nushell operations -// Polars LazyFrame is behind and Option to allow easy implementation of -// the Deserialize trait -#[derive(Default)] -pub struct NuLazyFrame { - pub lazy: Option, - pub schema: Option, - pub from_eager: bool, -} - -// Mocked serialization of the LazyFrame object -impl Serialize for NuLazyFrame { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_none() - } -} - -// Mocked deserialization of the LazyFrame object -impl<'de> Deserialize<'de> for NuLazyFrame { - fn deserialize(_deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Ok(NuLazyFrame::default()) - } -} - -impl fmt::Debug for NuLazyFrame { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "NuLazyframe") - } -} - -// Referenced access to the real LazyFrame -impl AsRef for NuLazyFrame { - fn as_ref(&self) -> &polars::prelude::LazyFrame { - // The only case when there cannot be a lazy frame is if it is created - // using the default function or if created by deserializing something - self.lazy.as_ref().expect("there should always be a frame") - } -} - -impl AsMut for NuLazyFrame { - fn as_mut(&mut self) -> &mut polars::prelude::LazyFrame { - // The only case when there cannot be a lazy frame is if it is created - // using the default function or if created by deserializing something - self.lazy.as_mut().expect("there should always be a frame") - } -} - -impl From for NuLazyFrame { - fn from(lazy_frame: LazyFrame) -> Self { - Self { - lazy: Some(lazy_frame), - from_eager: false, - schema: None, - } - } -} - -impl NuLazyFrame { - pub fn new(from_eager: bool, lazy: LazyFrame) -> Self { - Self { - lazy: Some(lazy), - from_eager, - schema: None, - } - } - - pub fn from_dataframe(df: NuDataFrame) -> Self { - let lazy = df.as_ref().clone().lazy(); - Self { - lazy: Some(lazy), - from_eager: true, - schema: Some(df.as_ref().schema()), - } - } - - pub fn into_value(self, span: Span) -> Result { - if self.from_eager { - let df = self.collect(span)?; - Ok(Value::custom(Box::new(df), span)) - } else { - Ok(Value::custom(Box::new(self), span)) - } - } - - pub fn into_polars(self) -> LazyFrame { - self.lazy.expect("lazyframe cannot be none to convert") - } - - pub fn collect(self, span: Span) -> Result { - self.lazy - .expect("No empty lazy for collect") - .collect() - .map_err(|e| ShellError::GenericError { - error: "Error collecting lazy frame".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }) - .map(|df| NuDataFrame { - df, - from_lazy: !self.from_eager, - }) - } - - pub fn try_from_value(value: Value) -> Result { - if Self::can_downcast(&value) { - Ok(Self::get_lazy_df(value)?) - } else if NuDataFrame::can_downcast(&value) { - let df = NuDataFrame::try_from_value(value)?; - Ok(NuLazyFrame::from_dataframe(df)) - } else { - Err(ShellError::CantConvert { - to_type: "lazy or eager dataframe".into(), - from_type: value.get_type().to_string(), - span: value.span(), - help: None, - }) - } - } - - pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span)?; - Self::try_from_value(value) - } - - pub fn get_lazy_df(value: Value) -> Result { - let span = value.span(); - match value { - Value::Custom { val, .. } => match val.as_any().downcast_ref::() { - Some(expr) => Ok(Self { - lazy: expr.lazy.clone(), - from_eager: false, - schema: None, - }), - None => Err(ShellError::CantConvert { - to_type: "lazy frame".into(), - from_type: "non-dataframe".into(), - span, - help: None, - }), - }, - x => Err(ShellError::CantConvert { - to_type: "lazy frame".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }), - } - } - - pub fn can_downcast(value: &Value) -> bool { - if let Value::Custom { val, .. } = value { - val.as_any().downcast_ref::().is_some() - } else { - false - } - } - - pub fn apply_with_expr(self, expr: NuExpression, f: F) -> Self - where - F: Fn(LazyFrame, Expr) -> LazyFrame, - { - let df = self.lazy.expect("Lazy frame must not be empty to apply"); - let expr = expr.into_polars(); - let new_frame = f(df, expr); - - Self { - from_eager: self.from_eager, - lazy: Some(new_frame), - schema: None, - } - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/custom_value.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/custom_value.rs deleted file mode 100644 index 6ac6cc6046..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/custom_value.rs +++ /dev/null @@ -1,44 +0,0 @@ -use super::NuLazyGroupBy; -use nu_protocol::{record, CustomValue, ShellError, Span, Value}; - -// CustomValue implementation for NuDataFrame -impl CustomValue for NuLazyGroupBy { - fn typetag_name(&self) -> &'static str { - "lazygroupby" - } - - fn typetag_deserialize(&self) { - unimplemented!("typetag_deserialize") - } - - fn clone_value(&self, span: nu_protocol::Span) -> Value { - let cloned = NuLazyGroupBy { - group_by: self.group_by.clone(), - schema: self.schema.clone(), - from_eager: self.from_eager, - }; - - Value::custom(Box::new(cloned), span) - } - - fn type_name(&self) -> String { - self.typetag_name().to_string() - } - - fn to_base_value(&self, span: Span) -> Result { - Ok(Value::record( - record! { - "LazyGroupBy" => Value::string("apply aggregation to complete execution plan", span) - }, - span, - )) - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs deleted file mode 100644 index e1bcb30069..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs +++ /dev/null @@ -1,113 +0,0 @@ -mod custom_value; - -use core::fmt; -use nu_protocol::{PipelineData, ShellError, Span, Value}; -use polars::prelude::{LazyGroupBy, Schema}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -// Lazyframe wrapper for Nushell operations -// Polars LazyFrame is behind and Option to allow easy implementation of -// the Deserialize trait -#[derive(Default)] -pub struct NuLazyGroupBy { - pub group_by: Option, - pub schema: Option, - pub from_eager: bool, -} - -// Mocked serialization of the LazyFrame object -impl Serialize for NuLazyGroupBy { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_none() - } -} - -// Mocked deserialization of the LazyFrame object -impl<'de> Deserialize<'de> for NuLazyGroupBy { - fn deserialize(_deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Ok(NuLazyGroupBy::default()) - } -} - -impl fmt::Debug for NuLazyGroupBy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "NuLazyGroupBy") - } -} - -// Referenced access to the real LazyFrame -impl AsRef for NuLazyGroupBy { - fn as_ref(&self) -> &polars::prelude::LazyGroupBy { - // The only case when there cannot be a lazy frame is if it is created - // using the default function or if created by deserializing something - self.group_by - .as_ref() - .expect("there should always be a frame") - } -} - -impl AsMut for NuLazyGroupBy { - fn as_mut(&mut self) -> &mut polars::prelude::LazyGroupBy { - // The only case when there cannot be a lazy frame is if it is created - // using the default function or if created by deserializing something - self.group_by - .as_mut() - .expect("there should always be a frame") - } -} - -impl From for NuLazyGroupBy { - fn from(group_by: LazyGroupBy) -> Self { - Self { - group_by: Some(group_by), - from_eager: false, - schema: None, - } - } -} - -impl NuLazyGroupBy { - pub fn into_value(self, span: Span) -> Value { - Value::custom(Box::new(self), span) - } - - pub fn into_polars(self) -> LazyGroupBy { - self.group_by.expect("GroupBy cannot be none to convert") - } - - pub fn try_from_value(value: Value) -> Result { - let span = value.span(); - match value { - Value::Custom { val, .. } => match val.as_any().downcast_ref::() { - Some(group) => Ok(Self { - group_by: group.group_by.clone(), - schema: group.schema.clone(), - from_eager: group.from_eager, - }), - None => Err(ShellError::CantConvert { - to_type: "lazy groupby".into(), - from_type: "custom value".into(), - span, - help: None, - }), - }, - x => Err(ShellError::CantConvert { - to_type: "lazy groupby".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }), - } - } - - pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span)?; - Self::try_from_value(value) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_schema.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_schema.rs deleted file mode 100644 index 3c2f689b85..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_schema.rs +++ /dev/null @@ -1,376 +0,0 @@ -use nu_protocol::{ShellError, Span, Value}; -use polars::prelude::{DataType, Field, Schema, SchemaRef, TimeUnit}; -use std::sync::Arc; - -#[derive(Debug, Clone)] -pub struct NuSchema { - pub schema: SchemaRef, -} - -impl NuSchema { - pub fn new(schema: Schema) -> Self { - Self { - schema: Arc::new(schema), - } - } -} - -impl TryFrom<&Value> for NuSchema { - type Error = ShellError; - fn try_from(value: &Value) -> Result { - let schema = value_to_schema(value, Span::unknown())?; - Ok(Self::new(schema)) - } -} - -impl From for Value { - fn from(schema: NuSchema) -> Self { - fields_to_value(schema.schema.iter_fields(), Span::unknown()) - } -} - -impl From for SchemaRef { - fn from(val: NuSchema) -> Self { - Arc::clone(&val.schema) - } -} - -fn fields_to_value(fields: impl Iterator, span: Span) -> Value { - let record = fields - .map(|field| { - let col = field.name().to_string(); - let val = dtype_to_value(field.data_type(), span); - (col, val) - }) - .collect(); - - Value::record(record, Span::unknown()) -} - -fn dtype_to_value(dtype: &DataType, span: Span) -> Value { - match dtype { - DataType::Struct(fields) => fields_to_value(fields.iter().cloned(), span), - _ => Value::string(dtype.to_string().replace('[', "<").replace(']', ">"), span), - } -} - -fn value_to_schema(value: &Value, span: Span) -> Result { - let fields = value_to_fields(value, span)?; - let schema = Schema::from_iter(fields); - Ok(schema) -} - -fn value_to_fields(value: &Value, span: Span) -> Result, ShellError> { - let fields = value - .as_record()? - .into_iter() - .map(|(col, val)| match val { - Value::Record { .. } => { - let fields = value_to_fields(val, span)?; - let dtype = DataType::Struct(fields); - Ok(Field::new(col, dtype)) - } - _ => { - let dtype = str_to_dtype(&val.coerce_string()?, span)?; - Ok(Field::new(col, dtype)) - } - }) - .collect::, ShellError>>()?; - Ok(fields) -} - -pub fn str_to_dtype(dtype: &str, span: Span) -> Result { - match dtype { - "bool" => Ok(DataType::Boolean), - "u8" => Ok(DataType::UInt8), - "u16" => Ok(DataType::UInt16), - "u32" => Ok(DataType::UInt32), - "u64" => Ok(DataType::UInt64), - "i8" => Ok(DataType::Int8), - "i16" => Ok(DataType::Int16), - "i32" => Ok(DataType::Int32), - "i64" => Ok(DataType::Int64), - "f32" => Ok(DataType::Float32), - "f64" => Ok(DataType::Float64), - "str" => Ok(DataType::String), - "binary" => Ok(DataType::Binary), - "date" => Ok(DataType::Date), - "time" => Ok(DataType::Time), - "null" => Ok(DataType::Null), - "unknown" => Ok(DataType::Unknown), - "object" => Ok(DataType::Object("unknown", None)), - _ if dtype.starts_with("list") => { - let dtype = dtype - .trim_start_matches("list") - .trim_start_matches('<') - .trim_end_matches('>') - .trim(); - let dtype = str_to_dtype(dtype, span)?; - Ok(DataType::List(Box::new(dtype))) - } - _ if dtype.starts_with("datetime") => { - let dtype = dtype - .trim_start_matches("datetime") - .trim_start_matches('<') - .trim_end_matches('>'); - let mut split = dtype.split(','); - let next = split - .next() - .ok_or_else(|| ShellError::GenericError { - error: "Invalid polars data type".into(), - msg: "Missing time unit".into(), - span: Some(span), - help: None, - inner: vec![], - })? - .trim(); - let time_unit = str_to_time_unit(next, span)?; - let next = split - .next() - .ok_or_else(|| ShellError::GenericError { - error: "Invalid polars data type".into(), - msg: "Missing time zone".into(), - span: Some(span), - help: None, - inner: vec![], - })? - .trim(); - let timezone = if "*" == next { - None - } else { - Some(next.to_string()) - }; - Ok(DataType::Datetime(time_unit, timezone)) - } - _ if dtype.starts_with("duration") => { - let inner = dtype.trim_start_matches("duration<").trim_end_matches('>'); - let next = inner - .split(',') - .next() - .ok_or_else(|| ShellError::GenericError { - error: "Invalid polars data type".into(), - msg: "Missing time unit".into(), - span: Some(span), - help: None, - inner: vec![], - })? - .trim(); - let time_unit = str_to_time_unit(next, span)?; - Ok(DataType::Duration(time_unit)) - } - _ => Err(ShellError::GenericError { - error: "Invalid polars data type".into(), - msg: format!("Unknown type: {dtype}"), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn str_to_time_unit(ts_string: &str, span: Span) -> Result { - match ts_string { - "ms" => Ok(TimeUnit::Milliseconds), - "us" | "μs" => Ok(TimeUnit::Microseconds), - "ns" => Ok(TimeUnit::Nanoseconds), - _ => Err(ShellError::GenericError { - error: "Invalid polars data type".into(), - msg: "Invalid time unit".into(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -#[cfg(test)] -mod test { - - use nu_protocol::record; - - use super::*; - - #[test] - fn test_value_to_schema() { - let address = record! { - "street" => Value::test_string("str"), - "city" => Value::test_string("str"), - }; - - let value = Value::test_record(record! { - "name" => Value::test_string("str"), - "age" => Value::test_string("i32"), - "address" => Value::test_record(address) - }); - - let schema = value_to_schema(&value, Span::unknown()).unwrap(); - let expected = Schema::from_iter(vec![ - Field::new("name", DataType::String), - Field::new("age", DataType::Int32), - Field::new( - "address", - DataType::Struct(vec![ - Field::new("street", DataType::String), - Field::new("city", DataType::String), - ]), - ), - ]); - assert_eq!(schema, expected); - } - - #[test] - fn test_dtype_str_to_schema_simple_types() { - let dtype = "bool"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Boolean; - assert_eq!(schema, expected); - - let dtype = "u8"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::UInt8; - assert_eq!(schema, expected); - - let dtype = "u16"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::UInt16; - assert_eq!(schema, expected); - - let dtype = "u32"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::UInt32; - assert_eq!(schema, expected); - - let dtype = "u64"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::UInt64; - assert_eq!(schema, expected); - - let dtype = "i8"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Int8; - assert_eq!(schema, expected); - - let dtype = "i16"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Int16; - assert_eq!(schema, expected); - - let dtype = "i32"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Int32; - assert_eq!(schema, expected); - - let dtype = "i64"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Int64; - assert_eq!(schema, expected); - - let dtype = "str"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::String; - assert_eq!(schema, expected); - - let dtype = "binary"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Binary; - assert_eq!(schema, expected); - - let dtype = "date"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Date; - assert_eq!(schema, expected); - - let dtype = "time"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Time; - assert_eq!(schema, expected); - - let dtype = "null"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Null; - assert_eq!(schema, expected); - - let dtype = "unknown"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Unknown; - assert_eq!(schema, expected); - - let dtype = "object"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Object("unknown", None); - assert_eq!(schema, expected); - } - - #[test] - fn test_dtype_str_schema_datetime() { - let dtype = "datetime"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Datetime(TimeUnit::Milliseconds, None); - assert_eq!(schema, expected); - - let dtype = "datetime"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Datetime(TimeUnit::Microseconds, None); - assert_eq!(schema, expected); - - let dtype = "datetime<μs, *>"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Datetime(TimeUnit::Microseconds, None); - assert_eq!(schema, expected); - - let dtype = "datetime"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Datetime(TimeUnit::Nanoseconds, None); - assert_eq!(schema, expected); - - let dtype = "datetime"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Datetime(TimeUnit::Milliseconds, Some("UTC".into())); - assert_eq!(schema, expected); - - let dtype = "invalid"; - let schema = str_to_dtype(dtype, Span::unknown()); - assert!(schema.is_err()) - } - - #[test] - fn test_dtype_str_schema_duration() { - let dtype = "duration"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Duration(TimeUnit::Milliseconds); - assert_eq!(schema, expected); - - let dtype = "duration"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Duration(TimeUnit::Microseconds); - assert_eq!(schema, expected); - - let dtype = "duration<μs>"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Duration(TimeUnit::Microseconds); - assert_eq!(schema, expected); - - let dtype = "duration"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Duration(TimeUnit::Nanoseconds); - assert_eq!(schema, expected); - } - - #[test] - fn test_dtype_str_to_schema_list_types() { - let dtype = "list"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::List(Box::new(DataType::Int32)); - assert_eq!(schema, expected); - - let dtype = "list>"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::List(Box::new(DataType::Duration(TimeUnit::Milliseconds))); - assert_eq!(schema, expected); - - let dtype = "list>"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::List(Box::new(DataType::Datetime(TimeUnit::Milliseconds, None))); - assert_eq!(schema, expected); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/custom_value.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/custom_value.rs deleted file mode 100644 index e2b73bcef1..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/custom_value.rs +++ /dev/null @@ -1,41 +0,0 @@ -use super::NuWhen; -use nu_protocol::{CustomValue, ShellError, Span, Value}; - -// CustomValue implementation for NuDataFrame -impl CustomValue for NuWhen { - fn typetag_name(&self) -> &'static str { - "when" - } - - fn typetag_deserialize(&self) { - unimplemented!("typetag_deserialize") - } - - fn clone_value(&self, span: nu_protocol::Span) -> Value { - let cloned = self.clone(); - - Value::custom(Box::new(cloned), span) - } - - fn type_name(&self) -> String { - self.typetag_name().to_string() - } - - fn to_base_value(&self, span: Span) -> Result { - let val: String = match self { - NuWhen::Then(_) => "whenthen".into(), - NuWhen::ChainedThen(_) => "whenthenthen".into(), - }; - - let value = Value::string(val, span); - Ok(value) - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/mod.rs deleted file mode 100644 index b33cde7483..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/mod.rs +++ /dev/null @@ -1,77 +0,0 @@ -mod custom_value; - -use core::fmt; -use nu_protocol::{ShellError, Span, Value}; -use polars::prelude::{col, when, ChainedThen, Then}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -#[derive(Clone)] -pub enum NuWhen { - Then(Box), - ChainedThen(ChainedThen), -} - -// Mocked serialization of the LazyFrame object -impl Serialize for NuWhen { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_none() - } -} - -// Mocked deserialization of the LazyFrame object -impl<'de> Deserialize<'de> for NuWhen { - fn deserialize(_deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Ok(NuWhen::Then(Box::new(when(col("a")).then(col("b"))))) - } -} - -impl fmt::Debug for NuWhen { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "NuWhen") - } -} - -impl From for NuWhen { - fn from(then: Then) -> Self { - NuWhen::Then(Box::new(then)) - } -} - -impl From for NuWhen { - fn from(chained_when: ChainedThen) -> Self { - NuWhen::ChainedThen(chained_when) - } -} - -impl NuWhen { - pub fn into_value(self, span: Span) -> Value { - Value::custom(Box::new(self), span) - } - - pub fn try_from_value(value: Value) -> Result { - let span = value.span(); - match value { - Value::Custom { val, .. } => match val.as_any().downcast_ref::() { - Some(expr) => Ok(expr.clone()), - None => Err(ShellError::CantConvert { - to_type: "when expression".into(), - from_type: "non when expression".into(), - span, - help: None, - }), - }, - x => Err(ShellError::CantConvert { - to_type: "when expression".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }), - } - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/utils.rs b/crates/nu-cmd-dataframe/src/dataframe/values/utils.rs deleted file mode 100644 index 0dc43399a3..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/utils.rs +++ /dev/null @@ -1,86 +0,0 @@ -use nu_protocol::{ShellError, Span, Spanned, Value}; - -// Default value used when selecting rows from dataframe -pub const DEFAULT_ROWS: usize = 5; - -// Converts a Vec to a Vec> with a Span marking the whole -// location of the columns for error referencing -pub(crate) fn convert_columns( - columns: Vec, - span: Span, -) -> Result<(Vec>, Span), ShellError> { - // First column span - let mut col_span = columns - .first() - .ok_or_else(|| ShellError::GenericError { - error: "Empty column list".into(), - msg: "Empty list found for command".into(), - span: Some(span), - help: None, - inner: vec![], - }) - .map(|v| v.span())?; - - let res = columns - .into_iter() - .map(|value| { - let span = value.span(); - match value { - Value::String { val, .. } => { - col_span = col_span.merge(span); - Ok(Spanned { item: val, span }) - } - _ => Err(ShellError::GenericError { - error: "Incorrect column format".into(), - msg: "Only string as column name".into(), - span: Some(span), - help: None, - inner: vec![], - }), - } - }) - .collect::>, _>>()?; - - Ok((res, col_span)) -} - -// Converts a Vec to a Vec with a Span marking the whole -// location of the columns for error referencing -pub(crate) fn convert_columns_string( - columns: Vec, - span: Span, -) -> Result<(Vec, Span), ShellError> { - // First column span - let mut col_span = columns - .first() - .ok_or_else(|| ShellError::GenericError { - error: "Empty column list".into(), - msg: "Empty list found for command".into(), - span: Some(span), - help: None, - inner: vec![], - }) - .map(|v| v.span())?; - - let res = columns - .into_iter() - .map(|value| { - let span = value.span(); - match value { - Value::String { val, .. } => { - col_span = col_span.merge(span); - Ok(val) - } - _ => Err(ShellError::GenericError { - error: "Incorrect column format".into(), - msg: "Only string as column name".into(), - span: Some(span), - help: None, - inner: vec![], - }), - } - }) - .collect::, _>>()?; - - Ok((res, col_span)) -} diff --git a/crates/nu-cmd-dataframe/src/lib.rs b/crates/nu-cmd-dataframe/src/lib.rs deleted file mode 100644 index 7e7c8014c6..0000000000 --- a/crates/nu-cmd-dataframe/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[cfg(feature = "dataframe")] -pub mod dataframe; -#[cfg(feature = "dataframe")] -pub use dataframe::*; diff --git a/crates/nu-cmd-extra/Cargo.toml b/crates/nu-cmd-extra/Cargo.toml index 4c3dce09c7..9e6270163d 100644 --- a/crates/nu-cmd-extra/Cargo.toml +++ b/crates/nu-cmd-extra/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" license = "MIT" name = "nu-cmd-extra" repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-extra" -version = "0.93.1" +version = "0.94.1" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -13,13 +13,13 @@ version = "0.93.1" bench = false [dependencies] -nu-cmd-base = { path = "../nu-cmd-base", version = "0.93.1" } -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-json = { version = "0.93.1", path = "../nu-json" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-pretty-hex = { version = "0.93.1", path = "../nu-pretty-hex" } -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } -nu-utils = { path = "../nu-utils", version = "0.93.1" } +nu-cmd-base = { path = "../nu-cmd-base", version = "0.94.1" } +nu-engine = { path = "../nu-engine", version = "0.94.1" } +nu-json = { version = "0.94.1", path = "../nu-json" } +nu-parser = { path = "../nu-parser", version = "0.94.1" } +nu-pretty-hex = { version = "0.94.1", path = "../nu-pretty-hex" } +nu-protocol = { path = "../nu-protocol", version = "0.94.1" } +nu-utils = { path = "../nu-utils", version = "0.94.1" } # Potential dependencies for extras heck = { workspace = true } @@ -37,6 +37,6 @@ extra = ["default"] default = [] [dev-dependencies] -nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.1" } -nu-command = { path = "../nu-command", version = "0.93.1" } -nu-test-support = { path = "../nu-test-support", version = "0.93.1" } +nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.94.1" } +nu-command = { path = "../nu-command", version = "0.94.1" } +nu-test-support = { path = "../nu-test-support", version = "0.94.1" } \ No newline at end of file diff --git a/crates/nu-cmd-extra/src/extra/bits/bits_.rs b/crates/nu-cmd-extra/src/extra/bits/bits_.rs index 6767d3dd83..1190c01b4d 100644 --- a/crates/nu-cmd-extra/src/extra/bits/bits_.rs +++ b/crates/nu-cmd-extra/src/extra/bits/bits_.rs @@ -29,16 +29,6 @@ impl Command for Bits { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Bits.signature(), - &Bits.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-cmd-extra/src/extra/filters/roll/roll_.rs b/crates/nu-cmd-extra/src/extra/filters/roll/roll_.rs index 76e167a575..867bc2706e 100644 --- a/crates/nu-cmd-extra/src/extra/filters/roll/roll_.rs +++ b/crates/nu-cmd-extra/src/extra/filters/roll/roll_.rs @@ -33,16 +33,6 @@ impl Command for Roll { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Roll.signature(), - &Roll.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-cmd-extra/src/extra/strings/str_/case/str_.rs b/crates/nu-cmd-extra/src/extra/strings/str_/case/str_.rs index cf4537f046..fe6cb86324 100644 --- a/crates/nu-cmd-extra/src/extra/strings/str_/case/str_.rs +++ b/crates/nu-cmd-extra/src/extra/strings/str_/case/str_.rs @@ -29,16 +29,6 @@ impl Command for Str { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Str.signature(), - &Str.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-cmd-lang/Cargo.toml b/crates/nu-cmd-lang/Cargo.toml index 7592d4303b..f7b43521d1 100644 --- a/crates/nu-cmd-lang/Cargo.toml +++ b/crates/nu-cmd-lang/Cargo.toml @@ -6,28 +6,27 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-lang" edition = "2021" license = "MIT" name = "nu-cmd-lang" -version = "0.93.1" +version = "0.94.1" [lib] bench = false [dependencies] -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } -nu-utils = { path = "../nu-utils", version = "0.93.1" } +nu-engine = { path = "../nu-engine", version = "0.94.1" } +nu-parser = { path = "../nu-parser", version = "0.94.1" } +nu-protocol = { path = "../nu-protocol", version = "0.94.1" } +nu-utils = { path = "../nu-utils", version = "0.94.1" } itertools = { workspace = true } -shadow-rs = { version = "0.27", default-features = false } +shadow-rs = { version = "0.28", default-features = false } [build-dependencies] -shadow-rs = { version = "0.27", default-features = false } +shadow-rs = { version = "0.28", default-features = false } [features] mimalloc = [] which-support = [] trash-support = [] sqlite = [] -dataframe = [] static-link-openssl = [] -system-clipboard = [] +system-clipboard = [] \ No newline at end of file diff --git a/crates/nu-cmd-lang/README.md b/crates/nu-cmd-lang/README.md index 836fabf6f8..6c7298c17a 100644 --- a/crates/nu-cmd-lang/README.md +++ b/crates/nu-cmd-lang/README.md @@ -8,7 +8,6 @@ top of including: * nu-command * nu-cli -* nu-cmd-dataframe * nu-cmd-extra As time goes on and the nu language develops further in parallel with nushell we will be adding other command crates to the system. diff --git a/crates/nu-cmd-lang/src/core_commands/alias.rs b/crates/nu-cmd-lang/src/core_commands/alias.rs index f3603611e4..f14f4d5827 100644 --- a/crates/nu-cmd-lang/src/core_commands/alias.rs +++ b/crates/nu-cmd-lang/src/core_commands/alias.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Alias; @@ -29,8 +30,8 @@ impl Command for Alias { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn search_terms(&self) -> Vec<&str> { diff --git a/crates/nu-cmd-lang/src/core_commands/const_.rs b/crates/nu-cmd-lang/src/core_commands/const_.rs index 4076ae87c9..f780c5ada9 100644 --- a/crates/nu-cmd-lang/src/core_commands/const_.rs +++ b/crates/nu-cmd-lang/src/core_commands/const_.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Const; @@ -30,8 +31,8 @@ impl Command for Const { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn search_terms(&self) -> Vec<&str> { diff --git a/crates/nu-cmd-lang/src/core_commands/def.rs b/crates/nu-cmd-lang/src/core_commands/def.rs index 922ba78abb..eb1124da19 100644 --- a/crates/nu-cmd-lang/src/core_commands/def.rs +++ b/crates/nu-cmd-lang/src/core_commands/def.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Def; @@ -28,8 +29,8 @@ impl Command for Def { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/describe.rs b/crates/nu-cmd-lang/src/core_commands/describe.rs index 7d6d7f6f83..3d992f3f33 100644 --- a/crates/nu-cmd-lang/src/core_commands/describe.rs +++ b/crates/nu-cmd-lang/src/core_commands/describe.rs @@ -163,6 +163,8 @@ fn run( let description = match input { PipelineData::ByteStream(stream, ..) => { + let type_ = stream.type_().describe(); + let description = if options.detailed { let origin = match stream.source() { ByteStreamSource::Read(_) => "unknown", @@ -172,14 +174,14 @@ fn run( Value::record( record! { - "type" => Value::string("byte stream", head), + "type" => Value::string(type_, head), "origin" => Value::string(origin, head), "metadata" => metadata_to_value(metadata, head), }, head, ) } else { - Value::string("byte stream", head) + Value::string(type_, head) }; if !options.no_collect { diff --git a/crates/nu-cmd-lang/src/core_commands/export.rs b/crates/nu-cmd-lang/src/core_commands/export.rs index e5d3d45683..565e7895dc 100644 --- a/crates/nu-cmd-lang/src/core_commands/export.rs +++ b/crates/nu-cmd-lang/src/core_commands/export.rs @@ -1,4 +1,5 @@ use nu_engine::{command_prelude::*, get_full_help}; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct ExportCommand; @@ -23,8 +24,8 @@ impl Command for ExportCommand { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( @@ -34,17 +35,7 @@ impl Command for ExportCommand { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &ExportCommand.signature(), - &ExportCommand.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-lang/src/core_commands/export_alias.rs b/crates/nu-cmd-lang/src/core_commands/export_alias.rs index 14caddcc7a..4df335da44 100644 --- a/crates/nu-cmd-lang/src/core_commands/export_alias.rs +++ b/crates/nu-cmd-lang/src/core_commands/export_alias.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct ExportAlias; @@ -29,8 +30,8 @@ impl Command for ExportAlias { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn search_terms(&self) -> Vec<&str> { diff --git a/crates/nu-cmd-lang/src/core_commands/export_const.rs b/crates/nu-cmd-lang/src/core_commands/export_const.rs index 988db50b2a..631d85ad89 100644 --- a/crates/nu-cmd-lang/src/core_commands/export_const.rs +++ b/crates/nu-cmd-lang/src/core_commands/export_const.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct ExportConst; @@ -30,8 +31,8 @@ impl Command for ExportConst { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/export_def.rs b/crates/nu-cmd-lang/src/core_commands/export_def.rs index 93c5932efb..7a2d3949e1 100644 --- a/crates/nu-cmd-lang/src/core_commands/export_def.rs +++ b/crates/nu-cmd-lang/src/core_commands/export_def.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct ExportDef; @@ -28,8 +29,8 @@ impl Command for ExportDef { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/export_extern.rs b/crates/nu-cmd-lang/src/core_commands/export_extern.rs index 9ca756cf93..1a2ba4e5cb 100644 --- a/crates/nu-cmd-lang/src/core_commands/export_extern.rs +++ b/crates/nu-cmd-lang/src/core_commands/export_extern.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct ExportExtern; @@ -25,8 +26,8 @@ impl Command for ExportExtern { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/export_module.rs b/crates/nu-cmd-lang/src/core_commands/export_module.rs index fdbd143fb0..53a6ca750c 100644 --- a/crates/nu-cmd-lang/src/core_commands/export_module.rs +++ b/crates/nu-cmd-lang/src/core_commands/export_module.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct ExportModule; @@ -30,8 +31,8 @@ impl Command for ExportModule { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/export_use.rs b/crates/nu-cmd-lang/src/core_commands/export_use.rs index 2e4fd3f3e9..5ca96a899e 100644 --- a/crates/nu-cmd-lang/src/core_commands/export_use.rs +++ b/crates/nu-cmd-lang/src/core_commands/export_use.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct ExportUse; @@ -29,8 +30,8 @@ impl Command for ExportUse { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/extern_.rs b/crates/nu-cmd-lang/src/core_commands/extern_.rs index 71400dbb7c..496f104650 100644 --- a/crates/nu-cmd-lang/src/core_commands/extern_.rs +++ b/crates/nu-cmd-lang/src/core_commands/extern_.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Extern; @@ -25,8 +26,8 @@ impl Command for Extern { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/for_.rs b/crates/nu-cmd-lang/src/core_commands/for_.rs index 6f9391614e..387af45282 100644 --- a/crates/nu-cmd-lang/src/core_commands/for_.rs +++ b/crates/nu-cmd-lang/src/core_commands/for_.rs @@ -1,4 +1,5 @@ use nu_engine::{command_prelude::*, get_eval_block, get_eval_expression}; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct For; @@ -41,8 +42,8 @@ impl Command for For { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/hide.rs b/crates/nu-cmd-lang/src/core_commands/hide.rs index 2cfafa6c02..d52d2131c9 100644 --- a/crates/nu-cmd-lang/src/core_commands/hide.rs +++ b/crates/nu-cmd-lang/src/core_commands/hide.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Hide; @@ -31,8 +32,8 @@ This command is a parser keyword. For details, check: https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/let_.rs b/crates/nu-cmd-lang/src/core_commands/let_.rs index cc5504d8d6..f2da628c31 100644 --- a/crates/nu-cmd-lang/src/core_commands/let_.rs +++ b/crates/nu-cmd-lang/src/core_commands/let_.rs @@ -1,4 +1,5 @@ use nu_engine::{command_prelude::*, get_eval_block}; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Let; @@ -30,8 +31,8 @@ impl Command for Let { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn search_terms(&self) -> Vec<&str> { diff --git a/crates/nu-cmd-lang/src/core_commands/module.rs b/crates/nu-cmd-lang/src/core_commands/module.rs index 45641649ff..908c0764e5 100644 --- a/crates/nu-cmd-lang/src/core_commands/module.rs +++ b/crates/nu-cmd-lang/src/core_commands/module.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Module; @@ -30,8 +31,8 @@ impl Command for Module { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/mut_.rs b/crates/nu-cmd-lang/src/core_commands/mut_.rs index 60c4c146db..5db3c929af 100644 --- a/crates/nu-cmd-lang/src/core_commands/mut_.rs +++ b/crates/nu-cmd-lang/src/core_commands/mut_.rs @@ -1,4 +1,5 @@ use nu_engine::{command_prelude::*, get_eval_block}; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Mut; @@ -30,8 +31,8 @@ impl Command for Mut { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn search_terms(&self) -> Vec<&str> { diff --git a/crates/nu-cmd-lang/src/core_commands/overlay/command.rs b/crates/nu-cmd-lang/src/core_commands/overlay/command.rs index db502c0932..00ec7438ad 100644 --- a/crates/nu-cmd-lang/src/core_commands/overlay/command.rs +++ b/crates/nu-cmd-lang/src/core_commands/overlay/command.rs @@ -1,4 +1,5 @@ use nu_engine::{command_prelude::*, get_full_help}; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Overlay; @@ -25,8 +26,8 @@ impl Command for Overlay { You must use one of the following subcommands. Using this command as-is will only produce this help message."# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( @@ -36,16 +37,6 @@ impl Command for Overlay { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Overlay.signature(), - &[], - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-cmd-lang/src/core_commands/overlay/hide.rs b/crates/nu-cmd-lang/src/core_commands/overlay/hide.rs index c1b4a653bc..7ea84a2a91 100644 --- a/crates/nu-cmd-lang/src/core_commands/overlay/hide.rs +++ b/crates/nu-cmd-lang/src/core_commands/overlay/hide.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct OverlayHide; @@ -35,8 +36,8 @@ impl Command for OverlayHide { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/overlay/new.rs b/crates/nu-cmd-lang/src/core_commands/overlay/new.rs index 8f9a0e53ea..a571c37947 100644 --- a/crates/nu-cmd-lang/src/core_commands/overlay/new.rs +++ b/crates/nu-cmd-lang/src/core_commands/overlay/new.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct OverlayNew; @@ -33,8 +34,8 @@ This command is a parser keyword. For details, check: https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/overlay/use_.rs b/crates/nu-cmd-lang/src/core_commands/overlay/use_.rs index 13c3f711ad..e8b51fb59b 100644 --- a/crates/nu-cmd-lang/src/core_commands/overlay/use_.rs +++ b/crates/nu-cmd-lang/src/core_commands/overlay/use_.rs @@ -2,7 +2,7 @@ use nu_engine::{ command_prelude::*, find_in_dirs_env, get_dirs_var_from_call, get_eval_block, redirect_env, }; use nu_parser::trim_quotes_str; -use nu_protocol::ast::Expr; +use nu_protocol::{ast::Expr, engine::CommandType}; use std::path::Path; @@ -50,8 +50,8 @@ impl Command for OverlayUse { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/return_.rs b/crates/nu-cmd-lang/src/core_commands/return_.rs index 969456d005..478224079b 100644 --- a/crates/nu-cmd-lang/src/core_commands/return_.rs +++ b/crates/nu-cmd-lang/src/core_commands/return_.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Return; @@ -28,8 +29,8 @@ impl Command for Return { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/scope/command.rs b/crates/nu-cmd-lang/src/core_commands/scope/command.rs index da507f3159..98439226cf 100644 --- a/crates/nu-cmd-lang/src/core_commands/scope/command.rs +++ b/crates/nu-cmd-lang/src/core_commands/scope/command.rs @@ -1,4 +1,5 @@ use nu_engine::{command_prelude::*, get_full_help}; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Scope; @@ -19,8 +20,8 @@ impl Command for Scope { "Commands for getting info about what is in scope." } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( @@ -30,16 +31,6 @@ impl Command for Scope { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Scope.signature(), - &[], - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-cmd-lang/src/core_commands/use_.rs b/crates/nu-cmd-lang/src/core_commands/use_.rs index 32978d7e62..b0f3648304 100644 --- a/crates/nu-cmd-lang/src/core_commands/use_.rs +++ b/crates/nu-cmd-lang/src/core_commands/use_.rs @@ -1,7 +1,10 @@ use nu_engine::{ command_prelude::*, find_in_dirs_env, get_dirs_var_from_call, get_eval_block, redirect_env, }; -use nu_protocol::ast::{Expr, Expression}; +use nu_protocol::{ + ast::{Expr, Expression}, + engine::CommandType, +}; #[derive(Clone)] pub struct Use; @@ -40,8 +43,8 @@ This command is a parser keyword. For details, check: https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-lang/src/core_commands/version.rs b/crates/nu-cmd-lang/src/core_commands/version.rs index 7aaa72b38a..22ef1f2a08 100644 --- a/crates/nu-cmd-lang/src/core_commands/version.rs +++ b/crates/nu-cmd-lang/src/core_commands/version.rs @@ -175,11 +175,6 @@ fn features_enabled() -> Vec { names.push("sqlite".to_string()); } - #[cfg(feature = "dataframe")] - { - names.push("dataframe".to_string()); - } - #[cfg(feature = "static-link-openssl")] { names.push("static-link-openssl".to_string()); diff --git a/crates/nu-cmd-plugin/Cargo.toml b/crates/nu-cmd-plugin/Cargo.toml index 0d83c3a407..f461f6bf04 100644 --- a/crates/nu-cmd-plugin/Cargo.toml +++ b/crates/nu-cmd-plugin/Cargo.toml @@ -5,16 +5,16 @@ edition = "2021" license = "MIT" name = "nu-cmd-plugin" repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-plugin" -version = "0.93.1" +version = "0.94.1" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-path = { path = "../nu-path", version = "0.93.1" } -nu-protocol = { path = "../nu-protocol", version = "0.93.1", features = ["plugin"] } -nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.93.1" } +nu-engine = { path = "../nu-engine", version = "0.94.1" } +nu-path = { path = "../nu-path", version = "0.94.1" } +nu-protocol = { path = "../nu-protocol", version = "0.94.1", features = ["plugin"] } +nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.94.1" } itertools = { workspace = true } -[dev-dependencies] +[dev-dependencies] \ No newline at end of file diff --git a/crates/nu-cmd-plugin/src/commands/plugin/mod.rs b/crates/nu-cmd-plugin/src/commands/plugin/mod.rs index 87daa5a328..36590e9a8a 100644 --- a/crates/nu-cmd-plugin/src/commands/plugin/mod.rs +++ b/crates/nu-cmd-plugin/src/commands/plugin/mod.rs @@ -37,17 +37,7 @@ impl Command for PluginCommand { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &PluginCommand.signature(), - &PluginCommand.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-plugin/src/commands/plugin/use_.rs b/crates/nu-cmd-plugin/src/commands/plugin/use_.rs index e5997efcf0..3cfb28f28b 100644 --- a/crates/nu-cmd-plugin/src/commands/plugin/use_.rs +++ b/crates/nu-cmd-plugin/src/commands/plugin/use_.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct PluginUse; @@ -52,8 +53,8 @@ it was already previously registered with `plugin add`. vec!["add", "register", "scope"] } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-cmd-plugin/src/commands/register.rs b/crates/nu-cmd-plugin/src/commands/register.rs index 924ab00d62..2c10456db7 100644 --- a/crates/nu-cmd-plugin/src/commands/register.rs +++ b/crates/nu-cmd-plugin/src/commands/register.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Register; @@ -48,8 +49,8 @@ This command is a parser keyword. For details, check: vec!["add"] } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-color-config/Cargo.toml b/crates/nu-color-config/Cargo.toml index f183f5c054..34f0ded963 100644 --- a/crates/nu-color-config/Cargo.toml +++ b/crates/nu-color-config/Cargo.toml @@ -5,18 +5,18 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-color-confi edition = "2021" license = "MIT" name = "nu-color-config" -version = "0.93.1" +version = "0.94.1" [lib] bench = false [dependencies] -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-json = { path = "../nu-json", version = "0.93.1" } +nu-protocol = { path = "../nu-protocol", version = "0.94.1" } +nu-engine = { path = "../nu-engine", version = "0.94.1" } +nu-json = { path = "../nu-json", version = "0.94.1" } nu-ansi-term = { workspace = true } serde = { workspace = true, features = ["derive"] } [dev-dependencies] -nu-test-support = { path = "../nu-test-support", version = "0.93.1" } +nu-test-support = { path = "../nu-test-support", version = "0.94.1" } \ No newline at end of file diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 6010e16285..25665cfc33 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" license = "MIT" name = "nu-command" repository = "https://github.com/nushell/nushell/tree/main/crates/nu-command" -version = "0.93.1" +version = "0.94.1" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -13,21 +13,21 @@ version = "0.93.1" bench = false [dependencies] -nu-cmd-base = { path = "../nu-cmd-base", version = "0.93.1" } -nu-color-config = { path = "../nu-color-config", version = "0.93.1" } -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-glob = { path = "../nu-glob", version = "0.93.1" } -nu-json = { path = "../nu-json", version = "0.93.1" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-path = { path = "../nu-path", version = "0.93.1" } -nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.93.1" } -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } -nu-system = { path = "../nu-system", version = "0.93.1" } -nu-table = { path = "../nu-table", version = "0.93.1" } -nu-term-grid = { path = "../nu-term-grid", version = "0.93.1" } -nu-utils = { path = "../nu-utils", version = "0.93.1" } +nu-cmd-base = { path = "../nu-cmd-base", version = "0.94.1" } +nu-color-config = { path = "../nu-color-config", version = "0.94.1" } +nu-engine = { path = "../nu-engine", version = "0.94.1" } +nu-glob = { path = "../nu-glob", version = "0.94.1" } +nu-json = { path = "../nu-json", version = "0.94.1" } +nu-parser = { path = "../nu-parser", version = "0.94.1" } +nu-path = { path = "../nu-path", version = "0.94.1" } +nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.94.1" } +nu-protocol = { path = "../nu-protocol", version = "0.94.1" } +nu-system = { path = "../nu-system", version = "0.94.1" } +nu-table = { path = "../nu-table", version = "0.94.1" } +nu-term-grid = { path = "../nu-term-grid", version = "0.94.1" } +nu-utils = { path = "../nu-utils", version = "0.94.1" } nu-ansi-term = { workspace = true } -nuon = { path = "../nuon", version = "0.93.1" } +nuon = { path = "../nuon", version = "0.94.1" } alphanumeric-sort = { workspace = true } base64 = { workspace = true } @@ -99,7 +99,7 @@ uu_whoami = { workspace = true } uuid = { workspace = true, features = ["v4"] } v_htmlescape = { workspace = true } wax = { workspace = true } -which = { workspace = true, optional = true } +which = { workspace = true } unicode-width = { workspace = true } [target.'cfg(windows)'.dependencies] @@ -134,11 +134,11 @@ workspace = true plugin = ["nu-parser/plugin"] sqlite = ["rusqlite"] trash-support = ["trash"] -which-support = ["which"] +which-support = [] [dev-dependencies] -nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.1" } -nu-test-support = { path = "../nu-test-support", version = "0.93.1" } +nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.94.1" } +nu-test-support = { path = "../nu-test-support", version = "0.94.1" } dirs-next = { workspace = true } mockito = { workspace = true, default-features = false } @@ -146,3 +146,4 @@ quickcheck = { workspace = true } quickcheck_macros = { workspace = true } rstest = { workspace = true, default-features = false } pretty_assertions = { workspace = true } +tempfile = { workspace = true } \ No newline at end of file diff --git a/crates/nu-command/src/bytes/bytes_.rs b/crates/nu-command/src/bytes/bytes_.rs index f262e6a82e..82bf7c619b 100644 --- a/crates/nu-command/src/bytes/bytes_.rs +++ b/crates/nu-command/src/bytes/bytes_.rs @@ -29,16 +29,6 @@ impl Command for Bytes { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Bytes.signature(), - &Bytes.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/bytes/collect.rs b/crates/nu-command/src/bytes/collect.rs index 9cd34496e4..74ea3e5d14 100644 --- a/crates/nu-command/src/bytes/collect.rs +++ b/crates/nu-command/src/bytes/collect.rs @@ -1,3 +1,4 @@ +use itertools::Itertools; use nu_engine::command_prelude::*; #[derive(Clone, Copy)] @@ -35,46 +36,33 @@ impl Command for BytesCollect { input: PipelineData, ) -> Result { let separator: Option> = call.opt(engine_state, stack, 0)?; + + let span = call.head; + // input should be a list of binary data. - let mut output_binary = vec![]; - for value in input { - match value { - Value::Binary { mut val, .. } => { - output_binary.append(&mut val); - // manually concat - // TODO: make use of std::slice::Join when it's available in stable. - if let Some(sep) = &separator { - let mut work_sep = sep.clone(); - output_binary.append(&mut work_sep) - } - } - // Explicitly propagate errors instead of dropping them. - Value::Error { error, .. } => return Err(*error), - other => { - return Err(ShellError::OnlySupportsThisInputType { + let metadata = input.metadata(); + let iter = Itertools::intersperse( + input.into_iter_strict(span)?.map(move |value| { + // Everything is wrapped in Some in case there's a separator, so we can flatten + Some(match value { + // Explicitly propagate errors instead of dropping them. + Value::Error { error, .. } => Err(*error), + Value::Binary { val, .. } => Ok(val), + other => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "binary".into(), wrong_type: other.get_type().to_string(), - dst_span: call.head, + dst_span: span, src_span: other.span(), - }); - } - } - } + }), + }) + }), + Ok(separator).transpose(), + ) + .flatten(); - match separator { - None => Ok(Value::binary(output_binary, call.head).into_pipeline_data()), - Some(sep) => { - if output_binary.is_empty() { - Ok(Value::binary(output_binary, call.head).into_pipeline_data()) - } else { - // have push one extra separator in previous step, pop them out. - for _ in sep { - let _ = output_binary.pop(); - } - Ok(Value::binary(output_binary, call.head).into_pipeline_data()) - } - } - } + let output = ByteStream::from_result_iter(iter, span, None, ByteStreamType::Binary); + + Ok(PipelineData::ByteStream(output, metadata)) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/conversions/into/binary.rs b/crates/nu-command/src/conversions/into/binary.rs index 479b0fc7d7..8eb7715754 100644 --- a/crates/nu-command/src/conversions/into/binary.rs +++ b/crates/nu-command/src/conversions/into/binary.rs @@ -127,15 +127,18 @@ fn into_binary( let cell_paths = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - if let PipelineData::ByteStream(stream, ..) = input { - // TODO: in the future, we may want this to stream out, converting each to bytes - Ok(Value::binary(stream.into_bytes()?, head).into_pipeline_data()) + if let PipelineData::ByteStream(stream, metadata) = input { + // Just set the type - that should be good enough + Ok(PipelineData::ByteStream( + stream.with_type(ByteStreamType::Binary), + metadata, + )) } else { let args = Arguments { cell_paths, compact: call.has_flag(engine_state, stack, "compact")?, }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, head, engine_state.ctrlc.clone()) } } diff --git a/crates/nu-command/src/conversions/into/cell_path.rs b/crates/nu-command/src/conversions/into/cell_path.rs index 6da317abd3..c05dad57ba 100644 --- a/crates/nu-command/src/conversions/into/cell_path.rs +++ b/crates/nu-command/src/conversions/into/cell_path.rs @@ -103,7 +103,7 @@ fn into_cell_path(call: &Call, input: PipelineData) -> Result Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, int".into(), - wrong_type: "byte stream".into(), + wrong_type: stream.type_().describe().into(), dst_span: head, src_span: stream.span(), }), diff --git a/crates/nu-command/src/conversions/into/command.rs b/crates/nu-command/src/conversions/into/command.rs index 37bbbff02e..03b8e81c4a 100644 --- a/crates/nu-command/src/conversions/into/command.rs +++ b/crates/nu-command/src/conversions/into/command.rs @@ -29,16 +29,6 @@ impl Command for Into { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Into.signature(), - &[], - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/conversions/into/string.rs b/crates/nu-command/src/conversions/into/string.rs index eda4f7e5a5..7c7d69cf4e 100644 --- a/crates/nu-command/src/conversions/into/string.rs +++ b/crates/nu-command/src/conversions/into/string.rs @@ -156,9 +156,23 @@ fn string_helper( let cell_paths = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - if let PipelineData::ByteStream(stream, ..) = input { - // TODO: in the future, we may want this to stream out, converting each to bytes - Ok(Value::string(stream.into_string()?, head).into_pipeline_data()) + if let PipelineData::ByteStream(stream, metadata) = input { + // Just set the type - that should be good enough. There is no guarantee that the data + // within a string stream is actually valid UTF-8. But refuse to do it if it was already set + // to binary + if stream.type_().is_string_coercible() { + Ok(PipelineData::ByteStream( + stream.with_type(ByteStreamType::String), + metadata, + )) + } else { + Err(ShellError::CantConvert { + to_type: "string".into(), + from_type: "binary".into(), + span: stream.span(), + help: Some("try using the `decode` command".into()), + }) + } } else { let config = engine_state.get_config().clone(); let args = Arguments { diff --git a/crates/nu-command/src/date/date_.rs b/crates/nu-command/src/date/date_.rs index 158940cc2e..fd67e9c923 100644 --- a/crates/nu-command/src/date/date_.rs +++ b/crates/nu-command/src/date/date_.rs @@ -42,26 +42,6 @@ impl Command for Date { call: &Call, _input: PipelineData, ) -> Result { - date(engine_state, stack, call) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } - -fn date( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let head = call.head; - - Ok(Value::string( - get_full_help( - &Date.signature(), - &Date.examples(), - engine_state, - stack, - false, - ), - head, - ) - .into_pipeline_data()) -} diff --git a/crates/nu-command/src/debug/profile.rs b/crates/nu-command/src/debug/profile.rs index bd5de6041a..0cef979094 100644 --- a/crates/nu-command/src/debug/profile.rs +++ b/crates/nu-command/src/debug/profile.rs @@ -28,6 +28,7 @@ impl Command for DebugProfile { Some('v'), ) .switch("expr", "Collect expression types", Some('x')) + .switch("lines", "Collect line numbers", Some('l')) .named( "max-depth", SyntaxShape::Int, @@ -90,6 +91,7 @@ confusing the id/parent_id hierarchy. The --expr flag is helpful for investigati let collect_expanded_source = call.has_flag(engine_state, stack, "expanded-source")?; let collect_values = call.has_flag(engine_state, stack, "values")?; let collect_exprs = call.has_flag(engine_state, stack, "expr")?; + let collect_lines = call.has_flag(engine_state, stack, "lines")?; let max_depth = call .get_flag(engine_state, stack, "max-depth")? .unwrap_or(2); @@ -101,6 +103,7 @@ confusing the id/parent_id hierarchy. The --expr flag is helpful for investigati collect_expanded_source, collect_values, collect_exprs, + collect_lines, call.span(), ); @@ -118,14 +121,11 @@ confusing the id/parent_id hierarchy. The --expr flag is helpful for investigati let result = ClosureEvalOnce::new(engine_state, stack, closure).run_with_input(input); - // TODO: See eval_source() - match result { - Ok(pipeline_data) => { - let _ = pipeline_data.into_value(call.span()); - // pipeline_data.print(engine_state, caller_stack, true, false) - } - Err(_e) => (), // TODO: Report error - } + // Return potential errors + let pipeline_data = result?; + + // Collect the output + let _ = pipeline_data.into_value(call.span()); Ok(engine_state .deactivate_debugger() diff --git a/crates/nu-command/src/debug/view.rs b/crates/nu-command/src/debug/view.rs index 38a4efc2e7..4ef1c1c7e0 100644 --- a/crates/nu-command/src/debug/view.rs +++ b/crates/nu-command/src/debug/view.rs @@ -29,16 +29,6 @@ impl Command for View { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &View.signature(), - &View.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/debug/view_source.rs b/crates/nu-command/src/debug/view_source.rs index 974a92e1ee..54bca55956 100644 --- a/crates/nu-command/src/debug/view_source.rs +++ b/crates/nu-command/src/debug/view_source.rs @@ -55,7 +55,7 @@ impl Command for ViewSource { } } // gets vector of positionals. - else if let Some(block_id) = decl.get_block_id() { + else if let Some(block_id) = decl.block_id() { let block = engine_state.get_block(block_id); if let Some(block_span) = block.span { let contents = engine_state.get_span_contents(block_span); diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index ba55472e15..a23f9c4ef4 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -164,6 +164,9 @@ pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState { #[cfg(any( target_os = "android", target_os = "linux", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", target_os = "macos", target_os = "windows" ))] diff --git a/crates/nu-command/src/env/config/config_.rs b/crates/nu-command/src/env/config/config_.rs index 30285c5c9e..1cd6ef4621 100644 --- a/crates/nu-command/src/env/config/config_.rs +++ b/crates/nu-command/src/env/config/config_.rs @@ -29,17 +29,7 @@ impl Command for ConfigMeta { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &ConfigMeta.signature(), - &ConfigMeta.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } fn search_terms(&self) -> Vec<&str> { diff --git a/crates/nu-command/src/env/config/config_env.rs b/crates/nu-command/src/env/config/config_env.rs index 3c8dff487e..32bd7bba90 100644 --- a/crates/nu-command/src/env/config/config_env.rs +++ b/crates/nu-command/src/env/config/config_env.rs @@ -1,6 +1,7 @@ -use super::utils::gen_command; use nu_cmd_base::util::get_editor; use nu_engine::{command_prelude::*, env_to_strings}; +use nu_protocol::{process::ChildProcess, ByteStream}; +use nu_system::ForegroundChild; #[derive(Clone)] pub struct ConfigEnv; @@ -47,7 +48,7 @@ impl Command for ConfigEnv { engine_state: &EngineState, stack: &mut Stack, call: &Call, - input: PipelineData, + _input: PipelineData, ) -> Result { // `--default` flag handling if call.has_flag(engine_state, stack, "default")? { @@ -55,27 +56,59 @@ impl Command for ConfigEnv { return Ok(Value::string(nu_utils::get_default_env(), head).into_pipeline_data()); } - let env_vars_str = env_to_strings(engine_state, stack)?; - let nu_config = match engine_state.get_config_path("env-path") { - Some(path) => path, - None => { - return Err(ShellError::GenericError { - error: "Could not find $nu.env-path".into(), - msg: "Could not find $nu.env-path".into(), - span: None, - help: None, - inner: vec![], - }); - } + // Find the editor executable. + let (editor_name, editor_args) = get_editor(engine_state, stack, call.head)?; + let paths = nu_engine::env::path_str(engine_state, stack, call.head)?; + let cwd = engine_state.cwd(Some(stack))?; + let editor_executable = + crate::which(&editor_name, &paths, &cwd).ok_or(ShellError::ExternalCommand { + label: format!("`{editor_name}` not found"), + help: "Failed to find the editor executable".into(), + span: call.head, + })?; + + let Some(env_path) = engine_state.get_config_path("env-path") else { + return Err(ShellError::GenericError { + error: "Could not find $nu.env-path".into(), + msg: "Could not find $nu.env-path".into(), + span: None, + help: None, + inner: vec![], + }); }; + let env_path = env_path.to_string_lossy().to_string(); - let (item, config_args) = get_editor(engine_state, stack, call.head)?; + // Create the command. + let mut command = std::process::Command::new(editor_executable); - gen_command(call.head, nu_config, item, config_args, env_vars_str).run_with_input( - engine_state, - stack, - input, - true, - ) + // Configure PWD. + command.current_dir(cwd); + + // Configure environment variables. + let envs = env_to_strings(engine_state, stack)?; + command.env_clear(); + command.envs(envs); + + // Configure args. + command.arg(env_path); + command.args(editor_args); + + // Spawn the child process. On Unix, also put the child process to + // foreground if we're in an interactive session. + #[cfg(windows)] + let child = ForegroundChild::spawn(command)?; + #[cfg(unix)] + let child = ForegroundChild::spawn( + command, + engine_state.is_interactive, + &engine_state.pipeline_externals_state, + )?; + + // Wrap the output into a `PipelineData::ByteStream`. + let child = ChildProcess::new(child, None, false, call.head)?; + Ok(PipelineData::ByteStream( + ByteStream::child(child, call.head), + None, + )) } } diff --git a/crates/nu-command/src/env/config/config_nu.rs b/crates/nu-command/src/env/config/config_nu.rs index c43d8d0726..08695ca5bb 100644 --- a/crates/nu-command/src/env/config/config_nu.rs +++ b/crates/nu-command/src/env/config/config_nu.rs @@ -1,6 +1,7 @@ -use super::utils::gen_command; use nu_cmd_base::util::get_editor; use nu_engine::{command_prelude::*, env_to_strings}; +use nu_protocol::{process::ChildProcess, ByteStream}; +use nu_system::ForegroundChild; #[derive(Clone)] pub struct ConfigNu; @@ -51,7 +52,7 @@ impl Command for ConfigNu { engine_state: &EngineState, stack: &mut Stack, call: &Call, - input: PipelineData, + _input: PipelineData, ) -> Result { // `--default` flag handling if call.has_flag(engine_state, stack, "default")? { @@ -59,27 +60,59 @@ impl Command for ConfigNu { return Ok(Value::string(nu_utils::get_default_config(), head).into_pipeline_data()); } - let env_vars_str = env_to_strings(engine_state, stack)?; - let nu_config = match engine_state.get_config_path("config-path") { - Some(path) => path, - None => { - return Err(ShellError::GenericError { - error: "Could not find $nu.config-path".into(), - msg: "Could not find $nu.config-path".into(), - span: None, - help: None, - inner: vec![], - }); - } + // Find the editor executable. + let (editor_name, editor_args) = get_editor(engine_state, stack, call.head)?; + let paths = nu_engine::env::path_str(engine_state, stack, call.head)?; + let cwd = engine_state.cwd(Some(stack))?; + let editor_executable = + crate::which(&editor_name, &paths, &cwd).ok_or(ShellError::ExternalCommand { + label: format!("`{editor_name}` not found"), + help: "Failed to find the editor executable".into(), + span: call.head, + })?; + + let Some(config_path) = engine_state.get_config_path("config-path") else { + return Err(ShellError::GenericError { + error: "Could not find $nu.config-path".into(), + msg: "Could not find $nu.config-path".into(), + span: None, + help: None, + inner: vec![], + }); }; + let config_path = config_path.to_string_lossy().to_string(); - let (item, config_args) = get_editor(engine_state, stack, call.head)?; + // Create the command. + let mut command = std::process::Command::new(editor_executable); - gen_command(call.head, nu_config, item, config_args, env_vars_str).run_with_input( - engine_state, - stack, - input, - true, - ) + // Configure PWD. + command.current_dir(cwd); + + // Configure environment variables. + let envs = env_to_strings(engine_state, stack)?; + command.env_clear(); + command.envs(envs); + + // Configure args. + command.arg(config_path); + command.args(editor_args); + + // Spawn the child process. On Unix, also put the child process to + // foreground if we're in an interactive session. + #[cfg(windows)] + let child = ForegroundChild::spawn(command)?; + #[cfg(unix)] + let child = ForegroundChild::spawn( + command, + engine_state.is_interactive, + &engine_state.pipeline_externals_state, + )?; + + // Wrap the output into a `PipelineData::ByteStream`. + let child = ChildProcess::new(child, None, false, call.head)?; + Ok(PipelineData::ByteStream( + ByteStream::child(child, call.head), + None, + )) } } diff --git a/crates/nu-command/src/env/config/mod.rs b/crates/nu-command/src/env/config/mod.rs index 9cd93d2491..fa6a3b4ca3 100644 --- a/crates/nu-command/src/env/config/mod.rs +++ b/crates/nu-command/src/env/config/mod.rs @@ -2,7 +2,6 @@ mod config_; mod config_env; mod config_nu; mod config_reset; -mod utils; pub use config_::ConfigMeta; pub use config_env::ConfigEnv; pub use config_nu::ConfigNu; diff --git a/crates/nu-command/src/env/config/utils.rs b/crates/nu-command/src/env/config/utils.rs deleted file mode 100644 index 4b3ea43483..0000000000 --- a/crates/nu-command/src/env/config/utils.rs +++ /dev/null @@ -1,36 +0,0 @@ -use crate::ExternalCommand; -use nu_protocol::{OutDest, Span, Spanned}; -use std::{collections::HashMap, path::Path}; - -pub(crate) fn gen_command( - span: Span, - config_path: &Path, - item: String, - config_args: Vec, - env_vars_str: HashMap, -) -> ExternalCommand { - let name = Spanned { item, span }; - - let mut args = vec![Spanned { - item: config_path.to_string_lossy().to_string(), - span: Span::unknown(), - }]; - - let number_of_args = config_args.len() + 1; - - for arg in config_args { - args.push(Spanned { - item: arg, - span: Span::unknown(), - }) - } - - ExternalCommand { - name, - args, - arg_keep_raw: vec![false; number_of_args], - out: OutDest::Inherit, - err: OutDest::Inherit, - env_vars: env_vars_str, - } -} diff --git a/crates/nu-command/src/env/with_env.rs b/crates/nu-command/src/env/with_env.rs index ff66d21f49..46f7d1eb6e 100644 --- a/crates/nu-command/src/env/with_env.rs +++ b/crates/nu-command/src/env/with_env.rs @@ -1,6 +1,5 @@ use nu_engine::{command_prelude::*, eval_block}; use nu_protocol::{debugger::WithoutDebug, engine::Closure}; -use std::collections::HashMap; #[derive(Clone)] pub struct WithEnv; @@ -58,78 +57,14 @@ fn with_env( call: &Call, input: PipelineData, ) -> Result { - let variable: Value = call.req(engine_state, stack, 0)?; - + let env: Record = call.req(engine_state, stack, 0)?; let capture_block: Closure = call.req(engine_state, stack, 1)?; let block = engine_state.get_block(capture_block.block_id); let mut stack = stack.captures_to_stack_preserve_out_dest(capture_block.captures); - let mut env: HashMap = HashMap::new(); - - match &variable { - Value::List { vals: table, .. } => { - nu_protocol::report_error_new( - engine_state, - &ShellError::GenericError { - error: "Deprecated argument type".into(), - msg: "providing the variables to `with-env` as a list or single row table has been deprecated".into(), - span: Some(variable.span()), - help: Some("use the record form instead".into()), - inner: vec![], - }, - ); - if table.len() == 1 { - // single row([[X W]; [Y Z]]) - match &table[0] { - Value::Record { val, .. } => { - for (k, v) in &**val { - env.insert(k.to_string(), v.clone()); - } - } - x => { - return Err(ShellError::CantConvert { - to_type: "record".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }); - } - } - } else { - // primitive values([X Y W Z]) - for row in table.chunks(2) { - if row.len() == 2 { - env.insert(row[0].coerce_string()?, row[1].clone()); - } - if row.len() == 1 { - return Err(ShellError::IncorrectValue { - msg: format!("Missing value for $env.{}", row[0].coerce_string()?), - val_span: row[0].span(), - call_span: call.head, - }); - } - } - } - } - // when get object by `open x.json` or `from json` - Value::Record { val, .. } => { - for (k, v) in &**val { - env.insert(k.clone(), v.clone()); - } - } - x => { - return Err(ShellError::CantConvert { - to_type: "record".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }); - } - }; - // TODO: factor list of prohibited env vars into common place for prohibited in ["PWD", "FILE_PWD", "CURRENT_FILE"] { - if env.contains_key(prohibited) { + if env.contains(prohibited) { return Err(ShellError::AutomaticEnvVarSetManually { envvar_name: prohibited.into(), span: call.head, diff --git a/crates/nu-command/src/filesystem/open.rs b/crates/nu-command/src/filesystem/open.rs index 5fb8527511..842eaa5f4c 100644 --- a/crates/nu-command/src/filesystem/open.rs +++ b/crates/nu-command/src/filesystem/open.rs @@ -172,7 +172,7 @@ impl Command for Open { match converter { Some((converter_id, ext)) => { let decl = engine_state.get_decl(converter_id); - let command_output = if let Some(block_id) = decl.get_block_id() { + let command_output = if let Some(block_id) = decl.block_id() { let block = engine_state.get_block(block_id); eval_block(engine_state, stack, block, stream) } else { diff --git a/crates/nu-command/src/filesystem/save.rs b/crates/nu-command/src/filesystem/save.rs index 1be74665b2..ab5257fb01 100644 --- a/crates/nu-command/src/filesystem/save.rs +++ b/crates/nu-command/src/filesystem/save.rs @@ -393,7 +393,7 @@ fn convert_to_extension( ) -> Result { if let Some(decl_id) = engine_state.find_decl(format!("to {extension}").as_bytes(), &[]) { let decl = engine_state.get_decl(decl_id); - if let Some(block_id) = decl.get_block_id() { + if let Some(block_id) = decl.block_id() { let block = engine_state.get_block(block_id); let eval_block = get_eval_block(engine_state); eval_block(engine_state, stack, block, input) @@ -508,7 +508,7 @@ fn get_files( } fn stream_to_file( - mut source: impl Read, + source: impl Read, known_size: Option, ctrlc: Option>, mut file: File, @@ -555,7 +555,7 @@ fn stream_to_file( Ok(()) } } else { - copy_with_interrupt(&mut source, &mut file, span, ctrlc.as_deref())?; + copy_with_interrupt(source, file, span, ctrlc.as_deref())?; Ok(()) } } diff --git a/crates/nu-command/src/filesystem/touch.rs b/crates/nu-command/src/filesystem/touch.rs index 08843b11f0..0253e7e317 100644 --- a/crates/nu-command/src/filesystem/touch.rs +++ b/crates/nu-command/src/filesystem/touch.rs @@ -1,10 +1,9 @@ use filetime::FileTime; -#[allow(deprecated)] -use nu_engine::{command_prelude::*, current_dir}; +use nu_engine::command_prelude::*; use nu_path::expand_path_with; use nu_protocol::NuGlob; -use std::{fs::OpenOptions, path::Path, time::SystemTime}; +use std::{fs::OpenOptions, time::SystemTime}; use super::util::get_rest_for_glob_pattern; @@ -69,6 +68,8 @@ impl Command for Touch { let no_create: bool = call.has_flag(engine_state, stack, "no-create")?; let files: Vec> = get_rest_for_glob_pattern(engine_state, stack, call, 0)?; + let cwd = engine_state.cwd(Some(stack))?; + if files.is_empty() { return Err(ShellError::MissingParameter { param_name: "requires file paths".to_string(), @@ -86,7 +87,7 @@ impl Command for Touch { } if let Some(reference) = reference { - let reference_path = Path::new(&reference.item); + let reference_path = nu_path::expand_path_with(reference.item, &cwd, true); if !reference_path.exists() { return Err(ShellError::FileNotFoundCustom { msg: "Reference path not found".into(), @@ -114,9 +115,6 @@ impl Command for Touch { })?; } - #[allow(deprecated)] - let cwd = current_dir(engine_state, stack)?; - for glob in files { let path = expand_path_with(glob.item.as_ref(), &cwd, glob.item.is_expand()); diff --git a/crates/nu-command/src/filters/drop/column.rs b/crates/nu-command/src/filters/drop/column.rs index 01c13deee4..94c0308ea8 100644 --- a/crates/nu-command/src/filters/drop/column.rs +++ b/crates/nu-command/src/filters/drop/column.rs @@ -135,7 +135,7 @@ fn drop_cols( PipelineData::Empty => Ok(PipelineData::Empty), PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "table or record".into(), - wrong_type: "byte stream".into(), + wrong_type: stream.type_().describe().into(), dst_span: head, src_span: stream.span(), }), diff --git a/crates/nu-command/src/filters/first.rs b/crates/nu-command/src/filters/first.rs index e581c3e84d..97c5159b97 100644 --- a/crates/nu-command/src/filters/first.rs +++ b/crates/nu-command/src/filters/first.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use std::io::Read; #[derive(Clone)] pub struct First; @@ -170,12 +171,42 @@ fn first_helper( )) } } - PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary or range".into(), - wrong_type: "byte stream".into(), - dst_span: head, - src_span: stream.span(), - }), + PipelineData::ByteStream(stream, metadata) => { + if stream.type_().is_binary_coercible() { + let span = stream.span(); + if let Some(mut reader) = stream.reader() { + if return_single_element { + // Take a single byte + let mut byte = [0u8]; + if reader.read(&mut byte).err_span(span)? > 0 { + Ok(Value::int(byte[0] as i64, head).into_pipeline_data()) + } else { + Err(ShellError::AccessEmptyContent { span: head }) + } + } else { + // Just take 'rows' bytes off the stream, mimicking the binary behavior + Ok(PipelineData::ByteStream( + ByteStream::read( + reader.take(rows as u64), + head, + None, + ByteStreamType::Binary, + ), + metadata, + )) + } + } else { + Ok(PipelineData::Empty) + } + } else { + Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary or range".into(), + wrong_type: stream.type_().describe().into(), + dst_span: head, + src_span: stream.span(), + }) + } + } PipelineData::Empty => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), wrong_type: "null".into(), diff --git a/crates/nu-command/src/filters/insert.rs b/crates/nu-command/src/filters/insert.rs index e8794304c8..5f1380b2ac 100644 --- a/crates/nu-command/src/filters/insert.rs +++ b/crates/nu-command/src/filters/insert.rs @@ -261,8 +261,8 @@ fn insert( type_name: "empty pipeline".to_string(), span: head, }), - PipelineData::ByteStream(..) => Err(ShellError::IncompatiblePathAccess { - type_name: "byte stream".to_string(), + PipelineData::ByteStream(stream, ..) => Err(ShellError::IncompatiblePathAccess { + type_name: stream.type_().describe().into(), span: head, }), } diff --git a/crates/nu-command/src/filters/items.rs b/crates/nu-command/src/filters/items.rs index 6afc0bc536..ed30486bee 100644 --- a/crates/nu-command/src/filters/items.rs +++ b/crates/nu-command/src/filters/items.rs @@ -86,7 +86,7 @@ impl Command for Items { }), PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "record".into(), - wrong_type: "byte stream".into(), + wrong_type: stream.type_().describe().into(), dst_span: call.head, src_span: stream.span(), }), diff --git a/crates/nu-command/src/filters/last.rs b/crates/nu-command/src/filters/last.rs index 7530126c26..efb300db01 100644 --- a/crates/nu-command/src/filters/last.rs +++ b/crates/nu-command/src/filters/last.rs @@ -1,6 +1,5 @@ use nu_engine::command_prelude::*; - -use std::collections::VecDeque; +use std::{collections::VecDeque, io::Read}; #[derive(Clone)] pub struct Last; @@ -160,12 +159,47 @@ impl Command for Last { }), } } - PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary or range".into(), - wrong_type: "byte stream".into(), - dst_span: head, - src_span: stream.span(), - }), + PipelineData::ByteStream(stream, ..) => { + if stream.type_().is_binary_coercible() { + let span = stream.span(); + if let Some(mut reader) = stream.reader() { + // Have to be a bit tricky here, but just consume into a VecDeque that we + // shrink to fit each time + const TAKE: u64 = 8192; + let mut buf = VecDeque::with_capacity(rows + TAKE as usize); + loop { + let taken = std::io::copy(&mut (&mut reader).take(TAKE), &mut buf) + .err_span(span)?; + if buf.len() > rows { + buf.drain(..(buf.len() - rows)); + } + if taken < TAKE { + // This must be EOF. + if return_single_element { + if !buf.is_empty() { + return Ok( + Value::int(buf[0] as i64, head).into_pipeline_data() + ); + } else { + return Err(ShellError::AccessEmptyContent { span: head }); + } + } else { + return Ok(Value::binary(buf, head).into_pipeline_data()); + } + } + } + } else { + Ok(PipelineData::Empty) + } + } else { + Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary or range".into(), + wrong_type: stream.type_().describe().into(), + dst_span: head, + src_span: stream.span(), + }) + } + } PipelineData::Empty => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), wrong_type: "null".into(), diff --git a/crates/nu-command/src/filters/skip/skip_.rs b/crates/nu-command/src/filters/skip/skip_.rs index 9048b34a58..d293a743da 100644 --- a/crates/nu-command/src/filters/skip/skip_.rs +++ b/crates/nu-command/src/filters/skip/skip_.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use std::io::{self, Read}; #[derive(Clone)] pub struct Skip; @@ -12,6 +13,7 @@ impl Command for Skip { Signature::build(self.name()) .input_output_types(vec![ (Type::table(), Type::table()), + (Type::Binary, Type::Binary), ( Type::List(Box::new(Type::Any)), Type::List(Box::new(Type::Any)), @@ -51,6 +53,11 @@ impl Command for Skip { "editions" => Value::test_int(2021), })])), }, + Example { + description: "Skip 2 bytes of a binary value", + example: "0x[01 23 45 67] | skip 2", + result: Some(Value::test_binary(vec![0x45, 0x67])), + }, ] } fn run( @@ -87,12 +94,29 @@ impl Command for Skip { let ctrlc = engine_state.ctrlc.clone(); let input_span = input.span().unwrap_or(call.head); match input { - PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary or range".into(), - wrong_type: "byte stream".into(), - dst_span: call.head, - src_span: stream.span(), - }), + PipelineData::ByteStream(stream, metadata) => { + if stream.type_().is_binary_coercible() { + let span = stream.span(); + if let Some(mut reader) = stream.reader() { + // Copy the number of skipped bytes into the sink before proceeding + io::copy(&mut (&mut reader).take(n as u64), &mut io::sink()) + .err_span(span)?; + Ok(PipelineData::ByteStream( + ByteStream::read(reader, call.head, None, ByteStreamType::Binary), + metadata, + )) + } else { + Ok(PipelineData::Empty) + } + } else { + Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary or range".into(), + wrong_type: stream.type_().describe().into(), + dst_span: call.head, + src_span: stream.span(), + }) + } + } PipelineData::Value(Value::Binary { val, .. }, metadata) => { let bytes = val.into_iter().skip(n).collect::>(); Ok(Value::binary(bytes, input_span).into_pipeline_data_with_metadata(metadata)) diff --git a/crates/nu-command/src/filters/take/take_.rs b/crates/nu-command/src/filters/take/take_.rs index 12840aa8d6..7ebe22f914 100644 --- a/crates/nu-command/src/filters/take/take_.rs +++ b/crates/nu-command/src/filters/take/take_.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use std::io::Read; #[derive(Clone)] pub struct Take; @@ -78,12 +79,31 @@ impl Command for Take { stream.modify(|iter| iter.take(rows_desired)), metadata, )), - PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary or range".into(), - wrong_type: "byte stream".into(), - dst_span: head, - src_span: stream.span(), - }), + PipelineData::ByteStream(stream, metadata) => { + if stream.type_().is_binary_coercible() { + if let Some(reader) = stream.reader() { + // Just take 'rows' bytes off the stream, mimicking the binary behavior + Ok(PipelineData::ByteStream( + ByteStream::read( + reader.take(rows_desired as u64), + head, + None, + ByteStreamType::Binary, + ), + metadata, + )) + } else { + Ok(PipelineData::Empty) + } + } else { + Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary or range".into(), + wrong_type: stream.type_().describe().into(), + dst_span: head, + src_span: stream.span(), + }) + } + } PipelineData::Empty => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), wrong_type: "null".into(), diff --git a/crates/nu-command/src/filters/tee.rs b/crates/nu-command/src/filters/tee.rs index 936dee5c79..2251f6646a 100644 --- a/crates/nu-command/src/filters/tee.rs +++ b/crates/nu-command/src/filters/tee.rs @@ -1,7 +1,7 @@ use nu_engine::{command_prelude::*, get_eval_block_with_early_return}; use nu_protocol::{ byte_stream::copy_with_interrupt, engine::Closure, process::ChildPipe, ByteStream, - ByteStreamSource, OutDest, + ByteStreamSource, OutDest, PipelineMetadata, }; use std::{ io::{self, Read, Write}, @@ -104,9 +104,13 @@ use it in your pipeline."# if let PipelineData::ByteStream(stream, metadata) = input { let span = stream.span(); let ctrlc = engine_state.ctrlc.clone(); - let eval_block = { - let metadata = metadata.clone(); - move |stream| eval_block(PipelineData::ByteStream(stream, metadata)) + let type_ = stream.type_(); + + let info = StreamInfo { + span, + ctrlc: ctrlc.clone(), + type_, + metadata: metadata.clone(), }; match stream.into_source() { @@ -115,10 +119,11 @@ use it in your pipeline."# return stderr_misuse(span, head); } - let tee = IoTee::new(read, span, eval_block)?; + let tee_thread = spawn_tee(info, eval_block)?; + let tee = IoTee::new(read, tee_thread); Ok(PipelineData::ByteStream( - ByteStream::read(tee, span, ctrlc), + ByteStream::read(tee, span, ctrlc, type_), metadata, )) } @@ -127,44 +132,32 @@ use it in your pipeline."# return stderr_misuse(span, head); } - let tee = IoTee::new(file, span, eval_block)?; + let tee_thread = spawn_tee(info, eval_block)?; + let tee = IoTee::new(file, tee_thread); Ok(PipelineData::ByteStream( - ByteStream::read(tee, span, ctrlc), + ByteStream::read(tee, span, ctrlc, type_), metadata, )) } ByteStreamSource::Child(mut child) => { let stderr_thread = if use_stderr { let stderr_thread = if let Some(stderr) = child.stderr.take() { + let tee_thread = spawn_tee(info.clone(), eval_block)?; + let tee = IoTee::new(stderr, tee_thread); match stack.stderr() { OutDest::Pipe | OutDest::Capture => { - let tee = IoTee::new(stderr, span, eval_block)?; child.stderr = Some(ChildPipe::Tee(Box::new(tee))); - None + Ok(None) } - OutDest::Null => Some(tee_pipe_on_thread( - stderr, - io::sink(), - span, - ctrlc.as_ref(), - eval_block, - )?), - OutDest::Inherit => Some(tee_pipe_on_thread( - stderr, - io::stderr(), - span, - ctrlc.as_ref(), - eval_block, - )?), - OutDest::File(file) => Some(tee_pipe_on_thread( - stderr, - file.clone(), - span, - ctrlc.as_ref(), - eval_block, - )?), - } + OutDest::Null => copy_on_thread(tee, io::sink(), &info).map(Some), + OutDest::Inherit => { + copy_on_thread(tee, io::stderr(), &info).map(Some) + } + OutDest::File(file) => { + copy_on_thread(tee, file.clone(), &info).map(Some) + } + }? } else { None }; @@ -175,37 +168,29 @@ use it in your pipeline."# child.stdout = Some(stdout); Ok(()) } - OutDest::Null => { - copy_pipe(stdout, io::sink(), span, ctrlc.as_deref()) - } - OutDest::Inherit => { - copy_pipe(stdout, io::stdout(), span, ctrlc.as_deref()) - } - OutDest::File(file) => { - copy_pipe(stdout, file.as_ref(), span, ctrlc.as_deref()) - } + OutDest::Null => copy_pipe(stdout, io::sink(), &info), + OutDest::Inherit => copy_pipe(stdout, io::stdout(), &info), + OutDest::File(file) => copy_pipe(stdout, file.as_ref(), &info), }?; } stderr_thread } else { let stderr_thread = if let Some(stderr) = child.stderr.take() { + let info = info.clone(); match stack.stderr() { OutDest::Pipe | OutDest::Capture => { child.stderr = Some(stderr); Ok(None) } OutDest::Null => { - copy_pipe_on_thread(stderr, io::sink(), span, ctrlc.as_ref()) - .map(Some) + copy_pipe_on_thread(stderr, io::sink(), &info).map(Some) } OutDest::Inherit => { - copy_pipe_on_thread(stderr, io::stderr(), span, ctrlc.as_ref()) - .map(Some) + copy_pipe_on_thread(stderr, io::stderr(), &info).map(Some) } OutDest::File(file) => { - copy_pipe_on_thread(stderr, file.clone(), span, ctrlc.as_ref()) - .map(Some) + copy_pipe_on_thread(stderr, file.clone(), &info).map(Some) } }? } else { @@ -213,29 +198,16 @@ use it in your pipeline."# }; if let Some(stdout) = child.stdout.take() { + let tee_thread = spawn_tee(info.clone(), eval_block)?; + let tee = IoTee::new(stdout, tee_thread); match stack.stdout() { OutDest::Pipe | OutDest::Capture => { - let tee = IoTee::new(stdout, span, eval_block)?; child.stdout = Some(ChildPipe::Tee(Box::new(tee))); Ok(()) } - OutDest::Null => { - tee_pipe(stdout, io::sink(), span, ctrlc.as_deref(), eval_block) - } - OutDest::Inherit => tee_pipe( - stdout, - io::stdout(), - span, - ctrlc.as_deref(), - eval_block, - ), - OutDest::File(file) => tee_pipe( - stdout, - file.as_ref(), - span, - ctrlc.as_deref(), - eval_block, - ), + OutDest::Null => copy(tee, io::sink(), &info), + OutDest::Inherit => copy(tee, io::stdout(), &info), + OutDest::File(file) => copy(tee, file.as_ref(), &info), }?; } @@ -350,7 +322,7 @@ where fn stderr_misuse(span: Span, head: Span) -> Result { Err(ShellError::UnsupportedInput { msg: "--stderr can only be used on external commands".into(), - input: "the input to `tee` is not an external commands".into(), + input: "the input to `tee` is not an external command".into(), msg_span: head, input_span: span, }) @@ -363,23 +335,12 @@ struct IoTee { } impl IoTee { - fn new( - reader: R, - span: Span, - eval_block: impl FnOnce(ByteStream) -> Result<(), ShellError> + Send + 'static, - ) -> Result { - let (sender, receiver) = mpsc::channel(); - - let thread = thread::Builder::new() - .name("tee".into()) - .spawn(move || eval_block(ByteStream::from_iter(receiver, span, None))) - .err_span(span)?; - - Ok(Self { + fn new(reader: R, tee: TeeThread) -> Self { + Self { reader, - sender: Some(sender), - thread: Some(thread), - }) + sender: Some(tee.sender), + thread: Some(tee.thread), + } } } @@ -411,68 +372,74 @@ impl Read for IoTee { } } -fn tee_pipe( - pipe: ChildPipe, - mut dest: impl Write, +struct TeeThread { + sender: Sender>, + thread: JoinHandle>, +} + +fn spawn_tee( + info: StreamInfo, + mut eval_block: impl FnMut(PipelineData) -> Result<(), ShellError> + Send + 'static, +) -> Result { + let (sender, receiver) = mpsc::channel(); + + let thread = thread::Builder::new() + .name("tee".into()) + .spawn(move || { + // We don't use ctrlc here because we assume it already has it on the other side + let stream = ByteStream::from_iter(receiver.into_iter(), info.span, None, info.type_); + eval_block(PipelineData::ByteStream(stream, info.metadata)) + }) + .err_span(info.span)?; + + Ok(TeeThread { sender, thread }) +} + +#[derive(Clone)] +struct StreamInfo { span: Span, - ctrlc: Option<&AtomicBool>, - eval_block: impl FnOnce(ByteStream) -> Result<(), ShellError> + Send + 'static, -) -> Result<(), ShellError> { - match pipe { - ChildPipe::Pipe(pipe) => { - let mut tee = IoTee::new(pipe, span, eval_block)?; - copy_with_interrupt(&mut tee, &mut dest, span, ctrlc)?; - } - ChildPipe::Tee(tee) => { - let mut tee = IoTee::new(tee, span, eval_block)?; - copy_with_interrupt(&mut tee, &mut dest, span, ctrlc)?; - } - } + ctrlc: Option>, + type_: ByteStreamType, + metadata: Option, +} + +fn copy(src: impl Read, dest: impl Write, info: &StreamInfo) -> Result<(), ShellError> { + copy_with_interrupt(src, dest, info.span, info.ctrlc.as_deref())?; Ok(()) } -fn tee_pipe_on_thread( - pipe: ChildPipe, +fn copy_pipe(pipe: ChildPipe, dest: impl Write, info: &StreamInfo) -> Result<(), ShellError> { + match pipe { + ChildPipe::Pipe(pipe) => copy(pipe, dest, info), + ChildPipe::Tee(tee) => copy(tee, dest, info), + } +} + +fn copy_on_thread( + src: impl Read + Send + 'static, dest: impl Write + Send + 'static, - span: Span, - ctrlc: Option<&Arc>, - eval_block: impl FnOnce(ByteStream) -> Result<(), ShellError> + Send + 'static, + info: &StreamInfo, ) -> Result>, ShellError> { - let ctrlc = ctrlc.cloned(); + let span = info.span; + let ctrlc = info.ctrlc.clone(); thread::Builder::new() - .name("stderr tee".into()) - .spawn(move || tee_pipe(pipe, dest, span, ctrlc.as_deref(), eval_block)) + .name("stderr copier".into()) + .spawn(move || { + copy_with_interrupt(src, dest, span, ctrlc.as_deref())?; + Ok(()) + }) .map_err(|e| e.into_spanned(span).into()) } -fn copy_pipe( - pipe: ChildPipe, - mut dest: impl Write, - span: Span, - ctrlc: Option<&AtomicBool>, -) -> Result<(), ShellError> { - match pipe { - ChildPipe::Pipe(mut pipe) => { - copy_with_interrupt(&mut pipe, &mut dest, span, ctrlc)?; - } - ChildPipe::Tee(mut tee) => { - copy_with_interrupt(&mut tee, &mut dest, span, ctrlc)?; - } - } - Ok(()) -} - fn copy_pipe_on_thread( pipe: ChildPipe, dest: impl Write + Send + 'static, - span: Span, - ctrlc: Option<&Arc>, + info: &StreamInfo, ) -> Result>, ShellError> { - let ctrlc = ctrlc.cloned(); - thread::Builder::new() - .name("stderr copier".into()) - .spawn(move || copy_pipe(pipe, dest, span, ctrlc.as_deref())) - .map_err(|e| e.into_spanned(span).into()) + match pipe { + ChildPipe::Pipe(pipe) => copy_on_thread(pipe, dest, info), + ChildPipe::Tee(tee) => copy_on_thread(tee, dest, info), + } } #[test] diff --git a/crates/nu-command/src/filters/update.rs b/crates/nu-command/src/filters/update.rs index 0d914d2d8e..e724ae77ad 100644 --- a/crates/nu-command/src/filters/update.rs +++ b/crates/nu-command/src/filters/update.rs @@ -225,8 +225,8 @@ fn update( type_name: "empty pipeline".to_string(), span: head, }), - PipelineData::ByteStream(..) => Err(ShellError::IncompatiblePathAccess { - type_name: "byte stream".to_string(), + PipelineData::ByteStream(stream, ..) => Err(ShellError::IncompatiblePathAccess { + type_name: stream.type_().describe().into(), span: head, }), } diff --git a/crates/nu-command/src/filters/upsert.rs b/crates/nu-command/src/filters/upsert.rs index 4313addd89..e3678972fb 100644 --- a/crates/nu-command/src/filters/upsert.rs +++ b/crates/nu-command/src/filters/upsert.rs @@ -285,8 +285,8 @@ fn upsert( type_name: "empty pipeline".to_string(), span: head, }), - PipelineData::ByteStream(..) => Err(ShellError::IncompatiblePathAccess { - type_name: "byte stream".to_string(), + PipelineData::ByteStream(stream, ..) => Err(ShellError::IncompatiblePathAccess { + type_name: stream.type_().describe().into(), span: head, }), } diff --git a/crates/nu-command/src/filters/values.rs b/crates/nu-command/src/filters/values.rs index ed33ebf643..f6ff8cda2e 100644 --- a/crates/nu-command/src/filters/values.rs +++ b/crates/nu-command/src/filters/values.rs @@ -182,7 +182,7 @@ fn values( } PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "record or table".into(), - wrong_type: "byte stream".into(), + wrong_type: stream.type_().describe().into(), dst_span: head, src_span: stream.span(), }), diff --git a/crates/nu-command/src/formats/from/command.rs b/crates/nu-command/src/formats/from/command.rs index ce5987e5b1..40085b51d2 100644 --- a/crates/nu-command/src/formats/from/command.rs +++ b/crates/nu-command/src/formats/from/command.rs @@ -29,16 +29,6 @@ impl Command for From { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &From.signature(), - &From.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/formats/from/delimited.rs b/crates/nu-command/src/formats/from/delimited.rs index 1fdea19482..853f3bd83e 100644 --- a/crates/nu-command/src/formats/from/delimited.rs +++ b/crates/nu-command/src/formats/from/delimited.rs @@ -1,7 +1,14 @@ use csv::{ReaderBuilder, Trim}; -use nu_protocol::{IntoPipelineData, PipelineData, ShellError, Span, Value}; +use nu_protocol::{ByteStream, ListStream, PipelineData, ShellError, Span, Value}; -fn from_delimited_string_to_value( +fn from_csv_error(err: csv::Error, span: Span) -> ShellError { + ShellError::DelimiterError { + msg: err.to_string(), + span, + } +} + +fn from_delimited_stream( DelimitedReaderConfig { separator, comment, @@ -12,9 +19,15 @@ fn from_delimited_string_to_value( no_infer, trim, }: DelimitedReaderConfig, - s: String, + input: ByteStream, span: Span, -) -> Result { +) -> Result { + let input_reader = if let Some(stream) = input.reader() { + stream + } else { + return Ok(ListStream::new(std::iter::empty(), span, None)); + }; + let mut reader = ReaderBuilder::new() .has_headers(!noheaders) .flexible(flexible) @@ -23,19 +36,29 @@ fn from_delimited_string_to_value( .quote(quote as u8) .escape(escape.map(|c| c as u8)) .trim(trim) - .from_reader(s.as_bytes()); + .from_reader(input_reader); let headers = if noheaders { - (1..=reader.headers()?.len()) + (1..=reader + .headers() + .map_err(|err| from_csv_error(err, span))? + .len()) .map(|i| format!("column{i}")) .collect::>() } else { - reader.headers()?.iter().map(String::from).collect() + reader + .headers() + .map_err(|err| from_csv_error(err, span))? + .iter() + .map(String::from) + .collect() }; - let mut rows = vec![]; - for row in reader.records() { - let row = row?; + let iter = reader.into_records().map(move |row| { + let row = match row { + Ok(row) => row, + Err(err) => return Value::error(from_csv_error(err, span), span), + }; let columns = headers.iter().cloned(); let values = row .into_iter() @@ -57,10 +80,10 @@ fn from_delimited_string_to_value( // // Otherwise, if there are less values than headers, // then `Value::nothing(span)` is used to fill the remaining columns. - rows.push(Value::record(columns.zip(values).collect(), span)); - } + Value::record(columns.zip(values).collect(), span) + }); - Ok(Value::list(rows, span)) + Ok(ListStream::new(iter, span, None)) } pub(super) struct DelimitedReaderConfig { @@ -79,14 +102,27 @@ pub(super) fn from_delimited_data( input: PipelineData, name: Span, ) -> Result { - let (concat_string, _span, metadata) = input.collect_string_strict(name)?; - - Ok(from_delimited_string_to_value(config, concat_string, name) - .map_err(|x| ShellError::DelimiterError { - msg: x.to_string(), - span: name, - })? - .into_pipeline_data_with_metadata(metadata)) + match input { + PipelineData::Empty => Ok(PipelineData::Empty), + PipelineData::Value(value, metadata) => { + let string = value.into_string()?; + let byte_stream = ByteStream::read_string(string, name, None); + Ok(PipelineData::ListStream( + from_delimited_stream(config, byte_stream, name)?, + metadata, + )) + } + PipelineData::ListStream(list_stream, _) => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "string".into(), + wrong_type: "list".into(), + dst_span: name, + src_span: list_stream.span(), + }), + PipelineData::ByteStream(byte_stream, metadata) => Ok(PipelineData::ListStream( + from_delimited_stream(config, byte_stream, name)?, + metadata, + )), + } } pub fn trim_from_str(trim: Option) -> Result { diff --git a/crates/nu-command/src/formats/from/json.rs b/crates/nu-command/src/formats/from/json.rs index ea449711c1..e8b5d58e9e 100644 --- a/crates/nu-command/src/formats/from/json.rs +++ b/crates/nu-command/src/formats/from/json.rs @@ -1,4 +1,10 @@ +use std::{ + io::{BufRead, Cursor}, + sync::{atomic::AtomicBool, Arc}, +}; + use nu_engine::command_prelude::*; +use nu_protocol::ListStream; #[derive(Clone)] pub struct FromJson; @@ -45,6 +51,15 @@ impl Command for FromJson { "b" => Value::test_int(2), })), }, + Example { + example: r#"'{ "a": 1 } +{ "b": 2 }' | from json --objects"#, + description: "Parse a stream of line-delimited JSON values", + result: Some(Value::test_list(vec![ + Value::test_record(record! {"a" => Value::test_int(1)}), + Value::test_record(record! {"b" => Value::test_int(2)}), + ])), + }, ] } @@ -56,49 +71,80 @@ impl Command for FromJson { input: PipelineData, ) -> Result { let span = call.head; - let (string_input, span, metadata) = input.collect_string_strict(span)?; - - if string_input.is_empty() { - return Ok(Value::nothing(span).into_pipeline_data()); - } let strict = call.has_flag(engine_state, stack, "strict")?; // TODO: turn this into a structured underline of the nu_json error if call.has_flag(engine_state, stack, "objects")? { - let lines = string_input.lines().filter(|line| !line.trim().is_empty()); - - let converted_lines: Vec<_> = if strict { - lines - .map(|line| { - convert_string_to_value_strict(line, span) - .unwrap_or_else(|err| Value::error(err, span)) - }) - .collect() - } else { - lines - .map(|line| { - convert_string_to_value(line, span) - .unwrap_or_else(|err| Value::error(err, span)) - }) - .collect() - }; - - Ok(converted_lines.into_pipeline_data_with_metadata( - span, - engine_state.ctrlc.clone(), - metadata, - )) - } else if strict { - Ok(convert_string_to_value_strict(&string_input, span)? - .into_pipeline_data_with_metadata(metadata)) + // Return a stream of JSON values, one for each non-empty line + match input { + PipelineData::Value(Value::String { val, .. }, metadata) => { + Ok(PipelineData::ListStream( + read_json_lines(Cursor::new(val), span, strict, engine_state.ctrlc.clone()), + metadata, + )) + } + PipelineData::ByteStream(stream, metadata) + if stream.type_() != ByteStreamType::Binary => + { + if let Some(reader) = stream.reader() { + Ok(PipelineData::ListStream( + read_json_lines(reader, span, strict, None), + metadata, + )) + } else { + Ok(PipelineData::Empty) + } + } + _ => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "string".into(), + wrong_type: input.get_type().to_string(), + dst_span: call.head, + src_span: input.span().unwrap_or(call.head), + }), + } } else { - Ok(convert_string_to_value(&string_input, span)? - .into_pipeline_data_with_metadata(metadata)) + // Return a single JSON value + let (string_input, span, metadata) = input.collect_string_strict(span)?; + + if string_input.is_empty() { + return Ok(Value::nothing(span).into_pipeline_data()); + } + + if strict { + Ok(convert_string_to_value_strict(&string_input, span)? + .into_pipeline_data_with_metadata(metadata)) + } else { + Ok(convert_string_to_value(&string_input, span)? + .into_pipeline_data_with_metadata(metadata)) + } } } } +/// Create a stream of values from a reader that produces line-delimited JSON +fn read_json_lines( + input: impl BufRead + Send + 'static, + span: Span, + strict: bool, + interrupt: Option>, +) -> ListStream { + let iter = input + .lines() + .filter(|line| line.as_ref().is_ok_and(|line| !line.trim().is_empty()) || line.is_err()) + .map(move |line| { + let line = line.err_span(span)?; + if strict { + convert_string_to_value_strict(&line, span) + } else { + convert_string_to_value(&line, span) + } + }) + .map(move |result| result.unwrap_or_else(|err| Value::error(err, span))); + + ListStream::new(iter, span, interrupt) +} + fn convert_nujson_to_value(value: nu_json::Value, span: Span) -> Value { match value { nu_json::Value::Array(array) => Value::list( diff --git a/crates/nu-command/src/formats/to/command.rs b/crates/nu-command/src/formats/to/command.rs index 26c9a259b6..2138085c87 100644 --- a/crates/nu-command/src/formats/to/command.rs +++ b/crates/nu-command/src/formats/to/command.rs @@ -29,16 +29,6 @@ impl Command for To { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &To.signature(), - &To.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/formats/to/csv.rs b/crates/nu-command/src/formats/to/csv.rs index 173c6fbd6b..e7786e60cc 100644 --- a/crates/nu-command/src/formats/to/csv.rs +++ b/crates/nu-command/src/formats/to/csv.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use crate::formats::to::delimited::to_delimited_data; use nu_engine::command_prelude::*; use nu_protocol::Config; @@ -27,26 +29,37 @@ impl Command for ToCsv { "do not output the columns names as the first row", Some('n'), ) + .named( + "columns", + SyntaxShape::List(SyntaxShape::String.into()), + "the names (in order) of the columns to use", + None, + ) .category(Category::Formats) } fn examples(&self) -> Vec { vec![ Example { - description: "Outputs an CSV string representing the contents of this table", + description: "Outputs a CSV string representing the contents of this table", example: "[[foo bar]; [1 2]] | to csv", result: Some(Value::test_string("foo,bar\n1,2\n")), }, Example { - description: "Outputs an CSV string representing the contents of this table", + description: "Outputs a CSV string representing the contents of this table", example: "[[foo bar]; [1 2]] | to csv --separator ';' ", result: Some(Value::test_string("foo;bar\n1;2\n")), }, Example { - description: "Outputs an CSV string representing the contents of this record", + description: "Outputs a CSV string representing the contents of this record", example: "{a: 1 b: 2} | to csv", result: Some(Value::test_string("a,b\n1,2\n")), }, + Example { + description: "Outputs a CSV stream with column names pre-determined", + example: "[[foo bar baz]; [1 2 3]] | to csv --columns [baz foo]", + result: Some(Value::test_string("baz,foo\n3,1\n")), + }, ] } @@ -64,8 +77,9 @@ impl Command for ToCsv { let head = call.head; let noheaders = call.has_flag(engine_state, stack, "noheaders")?; let separator: Option> = call.get_flag(engine_state, stack, "separator")?; - let config = engine_state.get_config(); - to_csv(input, noheaders, separator, head, config) + let columns: Option> = call.get_flag(engine_state, stack, "columns")?; + let config = engine_state.config.clone(); + to_csv(input, noheaders, separator, columns, head, config) } } @@ -73,13 +87,14 @@ fn to_csv( input: PipelineData, noheaders: bool, separator: Option>, + columns: Option>, head: Span, - config: &Config, + config: Arc, ) -> Result { let sep = match separator { Some(Spanned { item: s, span, .. }) => { if s == r"\t" { - '\t' + Spanned { item: '\t', span } } else { let vec_s: Vec = s.chars().collect(); if vec_s.len() != 1 { @@ -89,13 +104,19 @@ fn to_csv( span, }); }; - vec_s[0] + Spanned { + item: vec_s[0], + span: head, + } } } - _ => ',', + _ => Spanned { + item: ',', + span: head, + }, }; - to_delimited_data(noheaders, sep, "CSV", input, head, config) + to_delimited_data(noheaders, sep, columns, "CSV", input, head, config) } #[cfg(test)] diff --git a/crates/nu-command/src/formats/to/delimited.rs b/crates/nu-command/src/formats/to/delimited.rs index 490983d67b..a7a2480a34 100644 --- a/crates/nu-command/src/formats/to/delimited.rs +++ b/crates/nu-command/src/formats/to/delimited.rs @@ -1,113 +1,31 @@ -use csv::{Writer, WriterBuilder}; +use csv::WriterBuilder; use nu_cmd_base::formats::to::delimited::merge_descriptors; -use nu_protocol::{Config, IntoPipelineData, PipelineData, Record, ShellError, Span, Value}; -use std::{collections::VecDeque, error::Error}; +use nu_protocol::{ + ByteStream, ByteStreamType, Config, PipelineData, ShellError, Span, Spanned, Value, +}; +use std::{iter, sync::Arc}; -fn from_value_to_delimited_string( - value: &Value, - separator: char, - config: &Config, - head: Span, -) -> Result { - let span = value.span(); - match value { - Value::Record { val, .. } => record_to_delimited(val, span, separator, config, head), - Value::List { vals, .. } => table_to_delimited(vals, span, separator, config, head), - // Propagate errors by explicitly matching them before the final case. - Value::Error { error, .. } => Err(*error.clone()), - v => Err(make_unsupported_input_error(v, head, v.span())), - } -} - -fn record_to_delimited( - record: &Record, - span: Span, - separator: char, - config: &Config, - head: Span, -) -> Result { - let mut wtr = WriterBuilder::new() - .delimiter(separator as u8) - .from_writer(vec![]); - let mut fields: VecDeque = VecDeque::new(); - let mut values: VecDeque = VecDeque::new(); - - for (k, v) in record { - fields.push_back(k.clone()); - - values.push_back(to_string_tagged_value(v, config, head, span)?); - } - - wtr.write_record(fields).expect("can not write."); - wtr.write_record(values).expect("can not write."); - - writer_to_string(wtr).map_err(|_| make_conversion_error("record", span)) -} - -fn table_to_delimited( - vals: &[Value], - span: Span, - separator: char, - config: &Config, - head: Span, -) -> Result { - if let Some(val) = find_non_record(vals) { - return Err(make_unsupported_input_error(val, head, span)); - } - - let mut wtr = WriterBuilder::new() - .delimiter(separator as u8) - .from_writer(vec![]); - - let merged_descriptors = merge_descriptors(vals); - - if merged_descriptors.is_empty() { - let vals = vals - .iter() - .map(|ele| { - to_string_tagged_value(ele, config, head, span).unwrap_or_else(|_| String::new()) - }) - .collect::>(); - wtr.write_record(vals).expect("can not write"); - } else { - wtr.write_record(merged_descriptors.iter().map(|item| &item[..])) - .expect("can not write."); - - for l in vals { - // should always be true because of `find_non_record` above - if let Value::Record { val: l, .. } = l { - let mut row = vec![]; - for desc in &merged_descriptors { - row.push(match l.get(desc) { - Some(s) => to_string_tagged_value(s, config, head, span)?, - None => String::new(), - }); - } - wtr.write_record(&row).expect("can not write"); - } +fn make_csv_error(error: csv::Error, format_name: &str, head: Span) -> ShellError { + if let csv::ErrorKind::Io(error) = error.kind() { + ShellError::IOErrorSpanned { + msg: error.to_string(), + span: head, + } + } else { + ShellError::GenericError { + error: format!("Failed to generate {format_name} data"), + msg: error.to_string(), + span: Some(head), + help: None, + inner: vec![], } - } - writer_to_string(wtr).map_err(|_| make_conversion_error("table", span)) -} - -fn writer_to_string(writer: Writer>) -> Result> { - Ok(String::from_utf8(writer.into_inner()?)?) -} - -fn make_conversion_error(type_from: &str, span: Span) -> ShellError { - ShellError::CantConvert { - to_type: type_from.to_string(), - from_type: "string".to_string(), - span, - help: None, } } fn to_string_tagged_value( v: &Value, config: &Config, - span: Span, - head: Span, + format_name: &'static str, ) -> Result { match &v { Value::String { .. } @@ -123,50 +41,124 @@ fn to_string_tagged_value( Value::Nothing { .. } => Ok(String::new()), // Propagate existing errors Value::Error { error, .. } => Err(*error.clone()), - _ => Err(make_unsupported_input_error(v, head, span)), + _ => Err(make_cant_convert_error(v, format_name)), } } -fn make_unsupported_input_error(value: &Value, head: Span, span: Span) -> ShellError { +fn make_unsupported_input_error( + r#type: impl std::fmt::Display, + head: Span, + span: Span, +) -> ShellError { ShellError::UnsupportedInput { - msg: "Unexpected type".to_string(), - input: format!("input type: {:?}", value.get_type()), + msg: "expected table or record".to_string(), + input: format!("input type: {}", r#type), msg_span: head, input_span: span, } } -pub fn find_non_record(values: &[Value]) -> Option<&Value> { - values - .iter() - .find(|val| !matches!(val, Value::Record { .. })) +fn make_cant_convert_error(value: &Value, format_name: &'static str) -> ShellError { + ShellError::CantConvert { + to_type: "string".into(), + from_type: value.get_type().to_string(), + span: value.span(), + help: Some(format!( + "only simple values are supported for {format_name} output" + )), + } } pub fn to_delimited_data( noheaders: bool, - sep: char, + separator: Spanned, + columns: Option>, format_name: &'static str, input: PipelineData, - span: Span, - config: &Config, + head: Span, + config: Arc, ) -> Result { - let value = input.into_value(span)?; - let output = match from_value_to_delimited_string(&value, sep, config, span) { - Ok(mut x) => { - if noheaders { - if let Some(second_line) = x.find('\n') { - let start = second_line + 1; - x.replace_range(0..start, ""); - } - } - Ok(x) + let mut input = input; + let span = input.span().unwrap_or(head); + let metadata = input.metadata(); + + let separator = u8::try_from(separator.item).map_err(|_| ShellError::IncorrectValue { + msg: "separator must be an ASCII character".into(), + val_span: separator.span, + call_span: head, + })?; + + // Check to ensure the input is likely one of our supported types first. We can't check a stream + // without consuming it though + match input { + PipelineData::Value(Value::List { .. } | Value::Record { .. }, _) => (), + PipelineData::Value(Value::Error { error, .. }, _) => return Err(*error), + PipelineData::Value(other, _) => { + return Err(make_unsupported_input_error(other.get_type(), head, span)) } - Err(_) => Err(ShellError::CantConvert { - to_type: format_name.into(), - from_type: value.get_type().to_string(), - span: value.span(), - help: None, - }), - }?; - Ok(Value::string(output, span).into_pipeline_data()) + PipelineData::ByteStream(..) => { + return Err(make_unsupported_input_error("byte stream", head, span)) + } + PipelineData::ListStream(..) => (), + PipelineData::Empty => (), + } + + // Determine the columns we'll use. This is necessary even if we don't write the header row, + // because we need to write consistent columns. + let columns = match columns { + Some(columns) => columns, + None => { + // The columns were not provided. We need to detect them, and in order to do so, we have + // to convert the input into a value first, so that we can find all of them + let value = input.into_value(span)?; + let columns = match &value { + Value::List { vals, .. } => merge_descriptors(vals), + Value::Record { val, .. } => val.columns().cloned().collect(), + _ => return Err(make_unsupported_input_error(value.get_type(), head, span)), + }; + input = PipelineData::Value(value, metadata.clone()); + columns + } + }; + + // Generate a byte stream of all of the values in the pipeline iterator, with a non-strict + // iterator so we can still accept plain records. + let mut iter = input.into_iter(); + + // If we're configured to generate a header, we generate it first, then set this false + let mut is_header = !noheaders; + + let stream = ByteStream::from_fn(head, None, ByteStreamType::String, move |buffer| { + let mut wtr = WriterBuilder::new() + .delimiter(separator) + .from_writer(buffer); + + if is_header { + // Unless we are configured not to write a header, we write the header row now, once, + // before everything else. + wtr.write_record(&columns) + .map_err(|err| make_csv_error(err, format_name, head))?; + is_header = false; + Ok(true) + } else if let Some(row) = iter.next() { + // Write each column of a normal row, in order + let record = row.into_record()?; + for column in &columns { + let field = record + .get(column) + .map(|v| to_string_tagged_value(v, &config, format_name)) + .unwrap_or(Ok(String::new()))?; + wtr.write_field(field) + .map_err(|err| make_csv_error(err, format_name, head))?; + } + // End the row + wtr.write_record(iter::empty::()) + .map_err(|err| make_csv_error(err, format_name, head))?; + Ok(true) + } else { + Ok(false) + } + }); + + Ok(PipelineData::ByteStream(stream, metadata)) } diff --git a/crates/nu-command/src/formats/to/text.rs b/crates/nu-command/src/formats/to/text.rs index 7f1d632c13..fb240654f6 100644 --- a/crates/nu-command/src/formats/to/text.rs +++ b/crates/nu-command/src/formats/to/text.rs @@ -51,7 +51,12 @@ impl Command for ToText { str }); Ok(PipelineData::ByteStream( - ByteStream::from_iter(iter, span, engine_state.ctrlc.clone()), + ByteStream::from_iter( + iter, + span, + engine_state.ctrlc.clone(), + ByteStreamType::String, + ), meta, )) } diff --git a/crates/nu-command/src/formats/to/tsv.rs b/crates/nu-command/src/formats/to/tsv.rs index eeaeb6b401..edc9592409 100644 --- a/crates/nu-command/src/formats/to/tsv.rs +++ b/crates/nu-command/src/formats/to/tsv.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use crate::formats::to::delimited::to_delimited_data; use nu_engine::command_prelude::*; use nu_protocol::Config; @@ -21,6 +23,12 @@ impl Command for ToTsv { "do not output the column names as the first row", Some('n'), ) + .named( + "columns", + SyntaxShape::List(SyntaxShape::String.into()), + "the names (in order) of the columns to use", + None, + ) .category(Category::Formats) } @@ -31,15 +39,20 @@ impl Command for ToTsv { fn examples(&self) -> Vec { vec![ Example { - description: "Outputs an TSV string representing the contents of this table", + description: "Outputs a TSV string representing the contents of this table", example: "[[foo bar]; [1 2]] | to tsv", result: Some(Value::test_string("foo\tbar\n1\t2\n")), }, Example { - description: "Outputs an TSV string representing the contents of this record", + description: "Outputs a TSV string representing the contents of this record", example: "{a: 1 b: 2} | to tsv", result: Some(Value::test_string("a\tb\n1\t2\n")), }, + Example { + description: "Outputs a TSV stream with column names pre-determined", + example: "[[foo bar baz]; [1 2 3]] | to tsv --columns [baz foo]", + result: Some(Value::test_string("baz\tfoo\n3\t1\n")), + }, ] } @@ -52,18 +65,24 @@ impl Command for ToTsv { ) -> Result { let head = call.head; let noheaders = call.has_flag(engine_state, stack, "noheaders")?; - let config = engine_state.get_config(); - to_tsv(input, noheaders, head, config) + let columns: Option> = call.get_flag(engine_state, stack, "columns")?; + let config = engine_state.config.clone(); + to_tsv(input, noheaders, columns, head, config) } } fn to_tsv( input: PipelineData, noheaders: bool, + columns: Option>, head: Span, - config: &Config, + config: Arc, ) -> Result { - to_delimited_data(noheaders, '\t', "TSV", input, head, config) + let sep = Spanned { + item: '\t', + span: head, + }; + to_delimited_data(noheaders, sep, columns, "TSV", input, head, config) } #[cfg(test)] diff --git a/crates/nu-command/src/hash/hash_.rs b/crates/nu-command/src/hash/hash_.rs index e3b19624a2..d4eca79354 100644 --- a/crates/nu-command/src/hash/hash_.rs +++ b/crates/nu-command/src/hash/hash_.rs @@ -29,16 +29,6 @@ impl Command for Hash { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Self.signature(), - &Self.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/help/help_commands.rs b/crates/nu-command/src/help/help_commands.rs index bc0fd92d92..f2440cc36f 100644 --- a/crates/nu-command/src/help/help_commands.rs +++ b/crates/nu-command/src/help/help_commands.rs @@ -88,12 +88,13 @@ pub fn help_commands( } let output = engine_state - .get_signatures_with_examples(false) - .iter() - .filter(|(signature, _, _, _, _)| signature.name == name) - .map(|(signature, examples, _, _, is_parser_keyword)| { - get_full_help(signature, examples, engine_state, stack, *is_parser_keyword) + .get_decls_sorted(false) + .into_iter() + .filter_map(|(_, decl_id)| { + let decl = engine_state.get_decl(decl_id); + (decl.name() == name).then_some(decl) }) + .map(|cmd| get_full_help(cmd, engine_state, stack)) .collect::>(); if !output.is_empty() { diff --git a/crates/nu-command/src/help/help_externs.rs b/crates/nu-command/src/help/help_externs.rs index 22fb4a303c..4a5c8123a4 100644 --- a/crates/nu-command/src/help/help_externs.rs +++ b/crates/nu-command/src/help/help_externs.rs @@ -108,12 +108,13 @@ pub fn help_externs( } let output = engine_state - .get_signatures_with_examples(false) - .iter() - .filter(|(signature, _, _, _, _)| signature.name == name) - .map(|(signature, examples, _, _, is_parser_keyword)| { - get_full_help(signature, examples, engine_state, stack, *is_parser_keyword) + .get_decls_sorted(false) + .into_iter() + .filter_map(|(_, decl_id)| { + let decl = engine_state.get_decl(decl_id); + (decl.name() == name).then_some(decl) }) + .map(|cmd| get_full_help(cmd, engine_state, stack)) .collect::>(); if !output.is_empty() { diff --git a/crates/nu-command/src/help/help_modules.rs b/crates/nu-command/src/help/help_modules.rs index 690968251b..5b39133a6d 100644 --- a/crates/nu-command/src/help/help_modules.rs +++ b/crates/nu-command/src/help/help_modules.rs @@ -149,6 +149,7 @@ pub fn help_modules( if !module.decls.is_empty() || module.main.is_some() { let commands: Vec<(Vec, DeclId)> = engine_state .get_decls_sorted(false) + .into_iter() .filter(|(_, id)| !engine_state.get_decl(*id).is_alias()) .collect(); @@ -186,6 +187,7 @@ pub fn help_modules( if !module.decls.is_empty() { let aliases: Vec<(Vec, DeclId)> = engine_state .get_decls_sorted(false) + .into_iter() .filter(|(_, id)| engine_state.get_decl(*id).is_alias()) .collect(); diff --git a/crates/nu-command/src/math/math_.rs b/crates/nu-command/src/math/math_.rs index a4a146738f..2ac067af4e 100644 --- a/crates/nu-command/src/math/math_.rs +++ b/crates/nu-command/src/math/math_.rs @@ -29,16 +29,6 @@ impl Command for MathCommand { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &MathCommand.signature(), - &MathCommand.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/misc/source.rs b/crates/nu-command/src/misc/source.rs index 798b321c6b..08a979b9f5 100644 --- a/crates/nu-command/src/misc/source.rs +++ b/crates/nu-command/src/misc/source.rs @@ -1,4 +1,5 @@ use nu_engine::{command_prelude::*, get_eval_block_with_early_return}; +use nu_protocol::engine::CommandType; /// Source a file for environment variables. #[derive(Clone)] @@ -29,8 +30,8 @@ impl Command for Source { https://www.nushell.sh/book/thinking_in_nu.html"# } - fn is_parser_keyword(&self) -> bool { - true + fn command_type(&self) -> CommandType { + CommandType::Keyword } fn run( diff --git a/crates/nu-command/src/network/http/client.rs b/crates/nu-command/src/network/http/client.rs index 54f7749627..8317fb50bc 100644 --- a/crates/nu-command/src/network/http/client.rs +++ b/crates/nu-command/src/network/http/client.rs @@ -117,10 +117,20 @@ pub fn response_to_buffer( _ => None, }; + // Try to guess whether the response is definitely intended to binary or definitely intended to + // be UTF-8 text. Otherwise specify `None` and just guess. This doesn't have to be thorough. + let content_type_lowercase = response.header("content-type").map(|s| s.to_lowercase()); + let response_type = match content_type_lowercase.as_deref() { + Some("application/octet-stream") => ByteStreamType::Binary, + Some(h) if h.contains("charset=utf-8") => ByteStreamType::String, + _ => ByteStreamType::Unknown, + }; + let reader = response.into_reader(); PipelineData::ByteStream( - ByteStream::read(reader, span, engine_state.ctrlc.clone()).with_known_size(buffer_size), + ByteStream::read(reader, span, engine_state.ctrlc.clone(), response_type) + .with_known_size(buffer_size), None, ) } diff --git a/crates/nu-command/src/network/http/http_.rs b/crates/nu-command/src/network/http/http_.rs index 15bc96494c..361033708e 100644 --- a/crates/nu-command/src/network/http/http_.rs +++ b/crates/nu-command/src/network/http/http_.rs @@ -35,16 +35,6 @@ impl Command for Http { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Http.signature(), - &Http.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/network/url/url_.rs b/crates/nu-command/src/network/url/url_.rs index 9f795c7eab..9988063d46 100644 --- a/crates/nu-command/src/network/url/url_.rs +++ b/crates/nu-command/src/network/url/url_.rs @@ -33,16 +33,6 @@ impl Command for Url { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Url.signature(), - &Url.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/path/path_.rs b/crates/nu-command/src/path/path_.rs index 19351d590a..667008b658 100644 --- a/crates/nu-command/src/path/path_.rs +++ b/crates/nu-command/src/path/path_.rs @@ -42,16 +42,6 @@ the path literal."# call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &PathCommand.signature(), - &PathCommand.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/path/type.rs b/crates/nu-command/src/path/type.rs index 8fbc270445..e048679220 100644 --- a/crates/nu-command/src/path/type.rs +++ b/crates/nu-command/src/path/type.rs @@ -1,10 +1,11 @@ use super::PathSubcommandArguments; use nu_engine::command_prelude::*; -use nu_path::expand_tilde; use nu_protocol::engine::StateWorkingSet; -use std::path::Path; +use std::path::{Path, PathBuf}; -struct Arguments; +struct Arguments { + pwd: PathBuf, +} impl PathSubcommandArguments for Arguments {} @@ -45,19 +46,21 @@ If nothing is found, an empty string will be returned."# fn run( &self, engine_state: &EngineState, - _stack: &mut Stack, + stack: &mut Stack, call: &Call, input: PipelineData, ) -> Result { let head = call.head; - let args = Arguments; + let args = Arguments { + pwd: engine_state.cwd(Some(stack))?, + }; // This doesn't match explicit nulls if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } input.map( - move |value| super::operate(&r#type, &args, value, head), + move |value| super::operate(&path_type, &args, value, head), engine_state.ctrlc.clone(), ) } @@ -69,14 +72,16 @@ If nothing is found, an empty string will be returned."# input: PipelineData, ) -> Result { let head = call.head; - let args = Arguments; + let args = Arguments { + pwd: working_set.permanent().cwd(None)?, + }; // This doesn't match explicit nulls if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } input.map( - move |value| super::operate(&r#type, &args, value, head), + move |value| super::operate(&path_type, &args, value, head), working_set.permanent().ctrlc.clone(), ) } @@ -97,21 +102,12 @@ If nothing is found, an empty string will be returned."# } } -fn r#type(path: &Path, span: Span, _: &Arguments) -> Value { - let meta = if path.starts_with("~") { - let p = expand_tilde(path); - std::fs::symlink_metadata(p) - } else { - std::fs::symlink_metadata(path) - }; - - Value::string( - match &meta { - Ok(data) => get_file_type(data), - Err(_) => "", - }, - span, - ) +fn path_type(path: &Path, span: Span, args: &Arguments) -> Value { + let path = nu_path::expand_path_with(path, &args.pwd, true); + match std::fs::symlink_metadata(path) { + Ok(metadata) => Value::string(get_file_type(&metadata), span), + Err(err) => Value::error(err.into(), span), + } } fn get_file_type(md: &std::fs::Metadata) -> &str { diff --git a/crates/nu-command/src/random/random_.rs b/crates/nu-command/src/random/random_.rs index 21819fa24f..5cf14d7748 100644 --- a/crates/nu-command/src/random/random_.rs +++ b/crates/nu-command/src/random/random_.rs @@ -33,16 +33,6 @@ impl Command for RandomCommand { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &RandomCommand.signature(), - &RandomCommand.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/stor/stor_.rs b/crates/nu-command/src/stor/stor_.rs index e736fd8357..c5bb378c2d 100644 --- a/crates/nu-command/src/stor/stor_.rs +++ b/crates/nu-command/src/stor/stor_.rs @@ -29,16 +29,6 @@ impl Command for Stor { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Stor.signature(), - &Stor.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/strings/format/format_.rs b/crates/nu-command/src/strings/format/format_.rs index 18159b610f..21b46a8b05 100644 --- a/crates/nu-command/src/strings/format/format_.rs +++ b/crates/nu-command/src/strings/format/format_.rs @@ -29,16 +29,6 @@ impl Command for Format { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Format.signature(), - &Format.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/strings/split/command.rs b/crates/nu-command/src/strings/split/command.rs index cb52cdb44c..ff9057eacc 100644 --- a/crates/nu-command/src/strings/split/command.rs +++ b/crates/nu-command/src/strings/split/command.rs @@ -29,16 +29,6 @@ impl Command for SplitCommand { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &SplitCommand.signature(), - &SplitCommand.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/strings/str_/case/str_.rs b/crates/nu-command/src/strings/str_/case/str_.rs index cf4537f046..fe6cb86324 100644 --- a/crates/nu-command/src/strings/str_/case/str_.rs +++ b/crates/nu-command/src/strings/str_/case/str_.rs @@ -29,16 +29,6 @@ impl Command for Str { call: &Call, _input: PipelineData, ) -> Result { - Ok(Value::string( - get_full_help( - &Str.signature(), - &Str.examples(), - engine_state, - stack, - self.is_parser_keyword(), - ), - call.head, - ) - .into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } } diff --git a/crates/nu-command/src/strings/str_/index_of.rs b/crates/nu-command/src/strings/str_/index_of.rs index 457713c2df..df26df7493 100644 --- a/crates/nu-command/src/strings/str_/index_of.rs +++ b/crates/nu-command/src/strings/str_/index_of.rs @@ -405,7 +405,7 @@ mod tests { let range = Range::new( Value::int(0, Span::test_data()), Value::int(1, Span::test_data()), - Value::int(3, Span::test_data()), + Value::int(2, Span::test_data()), RangeInclusion::Inclusive, Span::test_data(), ) diff --git a/crates/nu-command/src/strings/str_/join.rs b/crates/nu-command/src/strings/str_/join.rs index 732434b20f..dd3a87dd61 100644 --- a/crates/nu-command/src/strings/str_/join.rs +++ b/crates/nu-command/src/strings/str_/join.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use std::io::Write; #[derive(Clone)] pub struct StrJoin; @@ -40,31 +41,40 @@ impl Command for StrJoin { ) -> Result { let separator: Option = call.opt(engine_state, stack, 0)?; - let config = engine_state.get_config(); + let config = engine_state.config.clone(); - // let output = input.collect_string(&separator.unwrap_or_default(), &config)?; - // Hmm, not sure what we actually want. - // `to_formatted_string` formats dates as human readable which feels funny. - let mut strings: Vec = vec![]; + let span = call.head; - for value in input { - let str = match value { - Value::Error { error, .. } => { - return Err(*error); + let metadata = input.metadata(); + let mut iter = input.into_iter(); + let mut first = true; + + let output = ByteStream::from_fn(span, None, ByteStreamType::String, move |buffer| { + // Write each input to the buffer + if let Some(value) = iter.next() { + // Write the separator if this is not the first + if first { + first = false; + } else if let Some(separator) = &separator { + write!(buffer, "{}", separator)?; } - Value::Date { val, .. } => format!("{val:?}"), - value => value.to_expanded_string("\n", config), - }; - strings.push(str); - } - let output = if let Some(separator) = separator { - strings.join(&separator) - } else { - strings.join("") - }; + match value { + Value::Error { error, .. } => { + return Err(*error); + } + // Hmm, not sure what we actually want. + // `to_expanded_string` formats dates as human readable which feels funny. + Value::Date { val, .. } => write!(buffer, "{val:?}")?, + value => write!(buffer, "{}", value.to_expanded_string("\n", &config))?, + } + Ok(true) + } else { + Ok(false) + } + }); - Ok(Value::string(output, call.head).into_pipeline_data()) + Ok(PipelineData::ByteStream(output, metadata)) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/strings/str_/substring.rs b/crates/nu-command/src/strings/str_/substring.rs index 55871401ef..4f5c953dda 100644 --- a/crates/nu-command/src/strings/str_/substring.rs +++ b/crates/nu-command/src/strings/str_/substring.rs @@ -70,7 +70,7 @@ impl Command for SubCommand { } fn usage(&self) -> &str { - "Get part of a string. Note that the start is included but the end is excluded, and that the first character of a string is index 0." + "Get part of a string. Note that the first character of a string is index 0." } fn search_terms(&self) -> Vec<&str> { @@ -108,12 +108,12 @@ impl Command for SubCommand { Example { description: "Get a substring \"nushell\" from the text \"good nushell\" using a range", - example: " 'good nushell' | str substring 5..12", + example: " 'good nushell' | str substring 5..11", result: Some(Value::test_string("nushell")), }, Example { description: "Count indexes and split using grapheme clusters", - example: " '🇯🇵ほげ ふが ぴよ' | str substring --grapheme-clusters 4..6", + example: " '🇯🇵ほげ ふが ぴよ' | str substring --grapheme-clusters 4..5", result: Some(Value::test_string("ふが")), }, ] diff --git a/crates/nu-command/src/system/exec.rs b/crates/nu-command/src/system/exec.rs index 21c98f121b..35ab9428be 100644 --- a/crates/nu-command/src/system/exec.rs +++ b/crates/nu-command/src/system/exec.rs @@ -1,7 +1,4 @@ -use super::run_external::create_external_command; -#[allow(deprecated)] -use nu_engine::{command_prelude::*, current_dir}; -use nu_protocol::OutDest; +use nu_engine::{command_prelude::*, env_to_strings}; #[derive(Clone)] pub struct Exec; @@ -35,7 +32,66 @@ On Windows based systems, Nushell will wait for the command to finish and then e call: &Call, _input: PipelineData, ) -> Result { - exec(engine_state, stack, call) + let cwd = engine_state.cwd(Some(stack))?; + + // Find the absolute path to the executable. If the command is not + // found, display a helpful error message. + let name: Spanned = call.req(engine_state, stack, 0)?; + let executable = { + let paths = nu_engine::env::path_str(engine_state, stack, call.head)?; + let Some(executable) = crate::which(&name.item, &paths, &cwd) else { + return Err(crate::command_not_found( + &name.item, + call.head, + engine_state, + stack, + )); + }; + executable + }; + + // Create the command. + let mut command = std::process::Command::new(executable); + + // Configure PWD. + command.current_dir(cwd); + + // Configure environment variables. + let envs = env_to_strings(engine_state, stack)?; + command.env_clear(); + command.envs(envs); + + // Configure args. + let args = crate::eval_arguments_from_call(engine_state, stack, call)?; + command.args(args.into_iter().map(|s| s.item)); + + // Execute the child process, replacing/terminating the current process + // depending on platform. + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + + let err = command.exec(); + Err(ShellError::ExternalCommand { + label: "Failed to exec into new process".into(), + help: err.to_string(), + span: call.head, + }) + } + #[cfg(windows)] + { + let mut child = command.spawn().map_err(|err| ShellError::ExternalCommand { + label: "Failed to exec into new process".into(), + help: err.to_string(), + span: call.head, + })?; + let status = child.wait().map_err(|err| ShellError::ExternalCommand { + label: "Failed to wait for child process".into(), + help: err.to_string(), + span: call.head, + })?; + std::process::exit(status.code().expect("status.code() succeeds on Windows")) + } } fn examples(&self) -> Vec { @@ -53,57 +109,3 @@ On Windows based systems, Nushell will wait for the command to finish and then e ] } } - -fn exec( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let mut external_command = create_external_command(engine_state, stack, call)?; - external_command.out = OutDest::Inherit; - external_command.err = OutDest::Inherit; - - #[allow(deprecated)] - let cwd = current_dir(engine_state, stack)?; - let mut command = external_command.spawn_simple_command(&cwd.to_string_lossy())?; - command.current_dir(cwd); - command.envs(external_command.env_vars); - - // this either replaces our process and should not return, - // or the exec fails and we get an error back - exec_impl(command, call.head) -} - -#[cfg(unix)] -fn exec_impl(mut command: std::process::Command, span: Span) -> Result { - use std::os::unix::process::CommandExt; - - let error = command.exec(); - - Err(ShellError::GenericError { - error: "Error on exec".into(), - msg: error.to_string(), - span: Some(span), - help: None, - inner: vec![], - }) -} - -#[cfg(windows)] -fn exec_impl(mut command: std::process::Command, span: Span) -> Result { - match command.spawn() { - Ok(mut child) => match child.wait() { - Ok(status) => std::process::exit(status.code().unwrap_or(0)), - Err(e) => Err(ShellError::ExternalCommand { - label: "Error in external command".into(), - help: e.to_string(), - span, - }), - }, - Err(e) => Err(ShellError::ExternalCommand { - label: "Error spawning external command".into(), - help: e.to_string(), - span, - }), - } -} diff --git a/crates/nu-command/src/system/mod.rs b/crates/nu-command/src/system/mod.rs index 788e1b7740..a87aac8a07 100644 --- a/crates/nu-command/src/system/mod.rs +++ b/crates/nu-command/src/system/mod.rs @@ -5,6 +5,8 @@ mod nu_check; target_os = "android", target_os = "linux", target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", target_os = "macos", target_os = "windows" ))] @@ -23,13 +25,15 @@ pub use nu_check::NuCheck; target_os = "android", target_os = "linux", target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", target_os = "macos", target_os = "windows" ))] pub use ps::Ps; #[cfg(windows)] pub use registry_query::RegistryQuery; -pub use run_external::{External, ExternalCommand}; +pub use run_external::{command_not_found, eval_arguments_from_call, which, External}; pub use sys::*; pub use uname::UName; pub use which_::Which; diff --git a/crates/nu-command/src/system/ps.rs b/crates/nu-command/src/system/ps.rs index 922bf7915b..c64549a44d 100644 --- a/crates/nu-command/src/system/ps.rs +++ b/crates/nu-command/src/system/ps.rs @@ -2,20 +2,13 @@ use itertools::Itertools; use nu_engine::command_prelude::*; -#[cfg(all( - unix, - not(target_os = "freebsd"), - not(target_os = "macos"), - not(target_os = "windows"), - not(target_os = "android"), -))] +#[cfg(target_os = "linux")] use procfs::WithCurrentSystemInfo; use std::time::Duration; #[derive(Clone)] pub struct Ps; -#[cfg(not(target_os = "freebsd"))] impl Command for Ps { fn name(&self) -> &str { "ps" @@ -82,7 +75,6 @@ impl Command for Ps { } } -#[cfg(not(target_os = "freebsd"))] fn run_ps( engine_state: &EngineState, stack: &mut Stack, @@ -111,12 +103,7 @@ fn run_ps( if long { record.push("command", Value::string(proc.command(), span)); - #[cfg(all( - unix, - not(target_os = "macos"), - not(target_os = "windows"), - not(target_os = "android"), - ))] + #[cfg(target_os = "linux")] { let proc_stat = proc .curr_proc diff --git a/crates/nu-command/src/system/run_external.rs b/crates/nu-command/src/system/run_external.rs index 2941d80de3..69c4a319bb 100644 --- a/crates/nu-command/src/system/run_external.rs +++ b/crates/nu-command/src/system/run_external.rs @@ -1,15 +1,18 @@ use nu_cmd_base::hook::eval_hook; use nu_engine::{command_prelude::*, env_to_strings, get_eval_expression}; -use nu_protocol::{ast::Expr, did_you_mean, process::ChildProcess, ByteStream, NuGlob, OutDest}; +use nu_protocol::{ + ast::{Expr, Expression}, + did_you_mean, + process::ChildProcess, + ByteStream, OutDest, +}; use nu_system::ForegroundChild; use nu_utils::IgnoreCaseExt; -use os_pipe::PipeReader; -use pathdiff::diff_paths; use std::{ - collections::HashMap, + borrow::Cow, io::Write, path::{Path, PathBuf}, - process::{Command as CommandSys, Stdio}, + process::Stdio, sync::Arc, thread, }; @@ -41,8 +44,152 @@ impl Command for External { call: &Call, input: PipelineData, ) -> Result { - let command = create_external_command(engine_state, stack, call)?; - command.run_with_input(engine_state, stack, input, false) + let cwd = engine_state.cwd(Some(stack))?; + + // Evaluate the command name in the same way the arguments are evaluated. Since this isn't + // a spread, it should return a one-element vec. + let name_expr = call + .positional_nth(0) + .ok_or_else(|| ShellError::MissingParameter { + param_name: "command".into(), + span: call.head, + })?; + let name = eval_argument(engine_state, stack, name_expr, false)? + .pop() + .expect("eval_argument returned zero-element vec") + .into_spanned(name_expr.span); + + // Find the absolute path to the executable. On Windows, set the + // executable to "cmd.exe" if it's is a CMD internal command. If the + // command is not found, display a helpful error message. + let executable = if cfg!(windows) && is_cmd_internal_command(&name.item) { + PathBuf::from("cmd.exe") + } else { + // Expand tilde on the name if it's a bare string (#13000) + let expanded_name = if is_bare_string(name_expr) { + expand_tilde(&name.item) + } else { + name.item.clone() + }; + + // Determine the PATH to be used and then use `which` to find it - though this has no + // effect if it's an absolute path already + let paths = nu_engine::env::path_str(engine_state, stack, call.head)?; + let Some(executable) = which(&expanded_name, &paths, &cwd) else { + return Err(command_not_found( + &name.item, + call.head, + engine_state, + stack, + )); + }; + executable + }; + + // Create the command. + let mut command = std::process::Command::new(executable); + + // Configure PWD. + command.current_dir(cwd); + + // Configure environment variables. + let envs = env_to_strings(engine_state, stack)?; + command.env_clear(); + command.envs(envs); + + // Configure args. + let args = eval_arguments_from_call(engine_state, stack, call)?; + #[cfg(windows)] + if is_cmd_internal_command(&name.item) { + use std::os::windows::process::CommandExt; + + // The /D flag disables execution of AutoRun commands from registry. + // The /C flag followed by a command name instructs CMD to execute + // that command and quit. + command.args(["/D", "/C", &name.item]); + for arg in &args { + command.raw_arg(escape_cmd_argument(arg)?.as_ref()); + } + } else { + command.args(args.into_iter().map(|s| s.item)); + } + #[cfg(not(windows))] + command.args(args.into_iter().map(|s| s.item)); + + // Configure stdout and stderr. If both are set to `OutDest::Pipe`, + // we'll setup a pipe that merge two streams into one. + let stdout = stack.stdout(); + let stderr = stack.stderr(); + let merged_stream = if matches!(stdout, OutDest::Pipe) && matches!(stderr, OutDest::Pipe) { + let (reader, writer) = os_pipe::pipe()?; + command.stdout(writer.try_clone()?); + command.stderr(writer); + Some(reader) + } else { + command.stdout(Stdio::try_from(stdout)?); + command.stderr(Stdio::try_from(stderr)?); + None + }; + + // Configure stdin. We'll try connecting input to the child process + // directly. If that's not possible, we'll setup a pipe and spawn a + // thread to copy data into the child process. + let data_to_copy_into_stdin = match input { + PipelineData::ByteStream(stream, metadata) => match stream.into_stdio() { + Ok(stdin) => { + command.stdin(stdin); + None + } + Err(stream) => { + command.stdin(Stdio::piped()); + Some(PipelineData::ByteStream(stream, metadata)) + } + }, + PipelineData::Empty => { + command.stdin(Stdio::inherit()); + None + } + value => { + command.stdin(Stdio::piped()); + Some(value) + } + }; + + // Spawn the child process. On Unix, also put the child process to + // foreground if we're in an interactive session. + #[cfg(windows)] + let mut child = ForegroundChild::spawn(command)?; + #[cfg(unix)] + let mut child = ForegroundChild::spawn( + command, + engine_state.is_interactive, + &engine_state.pipeline_externals_state, + )?; + + // If we need to copy data into the child process, do it now. + if let Some(data) = data_to_copy_into_stdin { + let stdin = child.as_mut().stdin.take().expect("stdin is piped"); + let engine_state = engine_state.clone(); + let stack = stack.clone(); + thread::Builder::new() + .name("external stdin worker".into()) + .spawn(move || { + let _ = write_pipeline_data(engine_state, stack, data, stdin); + }) + .err_span(call.head)?; + } + + // Wrap the output into a `PipelineData::ByteStream`. + let child = ChildProcess::new( + child, + merged_stream, + matches!(stderr, OutDest::Pipe), + call.head, + )?; + Ok(PipelineData::ByteStream( + ByteStream::child(child, call.head), + None, + )) } fn examples(&self) -> Vec { @@ -66,674 +213,525 @@ impl Command for External { } } -/// Creates ExternalCommand from a call -pub fn create_external_command( +/// Removes surrounding quotes from a string. Doesn't remove quotes from raw +/// strings. Returns the original string if it doesn't have matching quotes. +fn remove_quotes(s: &str) -> &str { + let quoted_by_double_quotes = s.len() >= 2 && s.starts_with('"') && s.ends_with('"'); + let quoted_by_single_quotes = s.len() >= 2 && s.starts_with('\'') && s.ends_with('\''); + let quoted_by_backticks = s.len() >= 2 && s.starts_with('`') && s.ends_with('`'); + if quoted_by_double_quotes || quoted_by_single_quotes || quoted_by_backticks { + &s[1..s.len() - 1] + } else { + s + } +} + +/// Evaluate all arguments from a call, performing expansions when necessary. +pub fn eval_arguments_from_call( engine_state: &EngineState, stack: &mut Stack, call: &Call, -) -> Result { - let name: Spanned = call.req(engine_state, stack, 0)?; - - // Translate environment variables from Values to Strings - let env_vars_str = env_to_strings(engine_state, stack)?; - - fn value_as_spanned(value: Value) -> Result, ShellError> { - let span = value.span(); - - value - .coerce_string() - .map(|item| Spanned { item, span }) - .map_err(|_| ShellError::ExternalCommand { - label: format!("Cannot convert {} to a string", value.get_type()), - help: "All arguments to an external command need to be string-compatible".into(), - span, - }) - } - - let eval_expression = get_eval_expression(engine_state); - - let mut spanned_args = vec![]; - let mut arg_keep_raw = vec![]; - for (arg, spread) in call.rest_iter(1) { - match eval_expression(engine_state, stack, arg)? { - Value::List { vals, .. } => { - if spread { - // turn all the strings in the array into params. - // Example: one_arg may be something like ["ls" "-a"] - // convert it to "ls" "-a" - for v in vals { - spanned_args.push(value_as_spanned(v)?); - // for arguments in list, it's always treated as a whole arguments - arg_keep_raw.push(true); - } - } else { - return Err(ShellError::CannotPassListToExternal { - arg: String::from_utf8_lossy(engine_state.get_span_contents(arg.span)) - .into(), - span: arg.span, - }); +) -> Result>, ShellError> { + let cwd = engine_state.cwd(Some(stack))?; + let mut args: Vec> = vec![]; + for (expr, spread) in call.rest_iter(1) { + if is_bare_string(expr) { + // If `expr` is a bare string, perform tilde-expansion, + // glob-expansion, and inner-quotes-removal, in that order. + for arg in eval_argument(engine_state, stack, expr, spread)? { + let tilde_expanded = expand_tilde(&arg); + for glob_expanded in expand_glob(&tilde_expanded, &cwd, expr.span)? { + let inner_quotes_removed = remove_inner_quotes(&glob_expanded); + args.push(inner_quotes_removed.into_owned().into_spanned(expr.span)); } } - val => { - if spread { - return Err(ShellError::CannotSpreadAsList { span: arg.span }); - } else { - spanned_args.push(value_as_spanned(val)?); - match arg.expr { - // refer to `parse_dollar_expr` function - // the expression type of $variable_name, $"($variable_name)" - // will be Expr::StringInterpolation, Expr::FullCellPath - Expr::StringInterpolation(_) | Expr::FullCellPath(_) => { - arg_keep_raw.push(true) - } - _ => arg_keep_raw.push(false), - } - } + } else { + for arg in eval_argument(engine_state, stack, expr, spread)? { + args.push(arg.into_spanned(expr.span)); } } } - - Ok(ExternalCommand { - name, - args: spanned_args, - arg_keep_raw, - out: stack.stdout().clone(), - err: stack.stderr().clone(), - env_vars: env_vars_str, - }) + Ok(args) } -#[derive(Clone)] -pub struct ExternalCommand { - pub name: Spanned, - pub args: Vec>, - pub arg_keep_raw: Vec, - pub out: OutDest, - pub err: OutDest, - pub env_vars: HashMap, -} - -impl ExternalCommand { - pub fn run_with_input( - &self, - engine_state: &EngineState, - stack: &mut Stack, - input: PipelineData, - reconfirm_command_name: bool, - ) -> Result { - let head = self.name.span; - - #[cfg(windows)] - let (child, reader, input) = { - // We may need to run `create_process` again, so we have to clone the underlying - // file or pipe in `input` here first. - let (input_consumed, stdin) = match &input { - PipelineData::ByteStream(stream, ..) => match stream.source() { - nu_protocol::ByteStreamSource::Read(_) => (false, Stdio::piped()), - nu_protocol::ByteStreamSource::File(file) => { - (true, file.try_clone().err_span(head)?.into()) - } - nu_protocol::ByteStreamSource::Child(child) => { - if let Some(nu_protocol::process::ChildPipe::Pipe(pipe)) = &child.stdout { - (true, pipe.try_clone().err_span(head)?.into()) - } else { - (false, Stdio::piped()) - } - } - }, - PipelineData::Empty => (false, Stdio::inherit()), - _ => (false, Stdio::piped()), - }; - - let mut input = input; - let (cmd, mut reader) = self.create_process(stdin, false, head)?; - let child = match ForegroundChild::spawn(cmd) { - Ok(child) => { - if input_consumed { - input = PipelineData::Empty; - } - Ok(child) - } - Err(err) => { - // Running external commands on Windows has 2 points of complication: - // 1. Some common Windows commands are actually built in to cmd.exe, not executables in their own right. - // 2. We need to let users run batch scripts etc. (.bat, .cmd) without typing their extension - - // To support these situations, we have a fallback path that gets run if a command - // fails to be run as a normal executable: - // 1. "shell out" to cmd.exe if the command is a known cmd.exe internal command - // 2. Otherwise, use `which-rs` to look for batch files etc. then run those in cmd.exe - - // set the default value, maybe we'll override it later - let mut child = Err(err); - - // This has the full list of cmd.exe "internal" commands: https://ss64.com/nt/syntax-internal.html - // I (Reilly) went through the full list and whittled it down to ones that are potentially useful: - const CMD_INTERNAL_COMMANDS: [&str; 9] = [ - "ASSOC", "CLS", "ECHO", "FTYPE", "MKLINK", "PAUSE", "START", "VER", "VOL", - ]; - let command_name = &self.name.item; - let looks_like_cmd_internal = CMD_INTERNAL_COMMANDS - .iter() - .any(|&cmd| command_name.eq_ignore_ascii_case(cmd)); - - let (data, stdin) = extract_stdio(input); - input = data; - - if looks_like_cmd_internal { - let (cmd, new_reader) = self.create_process(stdin, true, head)?; - reader = new_reader; - child = ForegroundChild::spawn(cmd); - } else { - #[cfg(feature = "which-support")] - { - // maybe it's a batch file (foo.cmd) and the user typed `foo`. Try to find it with `which-rs` - // TODO: clean this up with an if-let chain once those are stable - if let Ok(path) = - nu_engine::env::path_str(engine_state, stack, self.name.span) - { - if let Some(cwd) = self.env_vars.get("PWD") { - // append cwd to PATH so `which-rs` looks in the cwd too. - // this approximates what cmd.exe does. - let path_with_cwd = format!("{};{}", cwd, path); - if let Ok(which_path) = - which::which_in(&self.name.item, Some(path_with_cwd), cwd) - { - if let Some(file_name) = which_path.file_name() { - if !file_name - .to_string_lossy() - .eq_ignore_case(command_name) - { - // which-rs found an executable file with a slightly different name - // than the one the user tried. Let's try running it - let mut new_command = self.clone(); - new_command.name = Spanned { - item: file_name.to_string_lossy().to_string(), - span: self.name.span, - }; - let (cmd, new_reader) = new_command - .create_process(stdin, true, head)?; - reader = new_reader; - child = ForegroundChild::spawn(cmd); - } - } - } - } - } - } - } - - child - } - }; - - (child, reader, input) - }; - - #[cfg(unix)] - let (child, reader, input) = { - let (input, stdin) = extract_stdio(input); - let (cmd, reader) = self.create_process(stdin, false, head)?; - let child = ForegroundChild::spawn( - cmd, - engine_state.is_interactive, - &engine_state.pipeline_externals_state, - ); - (child, reader, input) - }; - - match child { - Err(err) => { - match err.kind() { - // If file not found, try suggesting alternative commands to the user - std::io::ErrorKind::NotFound => { - // recommend a replacement if the user tried a removed command - let command_name_lower = self.name.item.to_lowercase(); - let removed_from_nu = crate::removed_commands(); - if removed_from_nu.contains_key(&command_name_lower) { - let replacement = match removed_from_nu.get(&command_name_lower) { - Some(s) => s.clone(), - None => "".to_string(), - }; - return Err(ShellError::RemovedCommand { - removed: command_name_lower, - replacement, - span: self.name.span, - }); - } - - let suggestion = suggest_command(&self.name.item, engine_state); - let label = match suggestion { - Some(s) => { - if reconfirm_command_name { - format!( - "'{}' was not found; did you mean '{s}'?", - self.name.item - ) - } else { - let cmd_name = &self.name.item; - let maybe_module = engine_state - .which_module_has_decl(cmd_name.as_bytes(), &[]); - if let Some(module_name) = maybe_module { - let module_name = String::from_utf8_lossy(module_name); - let new_name = &[module_name.as_ref(), cmd_name].join(" "); - - if engine_state - .find_decl(new_name.as_bytes(), &[]) - .is_some() - { - format!("command '{cmd_name}' was not found but it was imported from module '{module_name}'; try using `{new_name}`") - } else { - format!("command '{cmd_name}' was not found but it exists in module '{module_name}'; try importing it with `use`") - } - } else { - // If command and suggestion are the same, display not found - if cmd_name == &s { - format!("'{cmd_name}' was not found") - } else { - format!("did you mean '{s}'?") - } - } - } - } - None => { - if reconfirm_command_name { - format!("executable '{}' was not found", self.name.item) - } else { - "executable was not found".into() - } - } - }; - - let mut err_str = err.to_string(); - if engine_state.is_interactive { - let mut engine_state = engine_state.clone(); - if let Some(hook) = engine_state.config.hooks.command_not_found.clone() - { - let canary = "ENTERED_COMMAND_NOT_FOUND"; - let stack = &mut stack.start_capture(); - if stack.has_env_var(&engine_state, canary) { - return Err(ShellError::ExternalCommand { - label: "command_not_found handler could not be run".into(), - help: "make sure the command_not_found closure itself does not use unknown commands".to_string(), - span: self.name.span, - }); - } - stack.add_env_var( - canary.to_string(), - Value::bool(true, Span::unknown()), - ); - match eval_hook( - &mut engine_state, - stack, - None, - vec![( - "cmd_name".into(), - Value::string(self.name.item.to_string(), self.name.span), - )], - &hook, - "command_not_found", - ) { - Ok(PipelineData::Value(Value::String { val, .. }, ..)) => { - err_str = format!("{}\n{}", err_str, val); - } - - Err(err) => { - stack.remove_env_var(&engine_state, canary); - return Err(err); - } - _ => {} - } - stack.remove_env_var(&engine_state, canary); - } - } - - Err(ShellError::ExternalCommand { - label, - help: err_str, - span: self.name.span, - }) - } - // otherwise, a default error message - _ => Err(ShellError::ExternalCommand { - label: "can't run executable".into(), - help: err.to_string(), - span: self.name.span, - }), - } - } - Ok(mut child) => { - if !input.is_nothing() { - let mut engine_state = engine_state.clone(); - let mut stack = stack.clone(); - - // Turn off color as we pass data through - Arc::make_mut(&mut engine_state.config).use_ansi_coloring = false; - - // Pipe input into the external command's stdin - if let Some(mut stdin_write) = child.as_mut().stdin.take() { - thread::Builder::new() - .name("external stdin worker".to_string()) - .spawn(move || { - let input = match input { - input @ PipelineData::ByteStream(..) => input, - input @ PipelineData::Value(Value::Binary { .. }, ..) => input, - input => { - let stack = &mut stack.start_capture(); - // Attempt to render the input as a table before piping it to the external. - // This is important for pagers like `less`; - // they need to get Nu data rendered for display to users. - // - // TODO: should we do something different for list inputs? - // Users often expect those to be piped to *nix tools as raw strings separated by newlines - crate::Table.run( - &engine_state, - stack, - &Call::new(head), - input, - )? - } - }; - - if let PipelineData::ByteStream(stream, ..) = input { - stream.write_to(&mut stdin_write)?; - } else { - for value in input.into_iter() { - let buf = value.coerce_into_binary()?; - stdin_write.write_all(&buf)?; - } - } - - Ok::<_, ShellError>(()) - }) - .err_span(head)?; - } - } - - let child = - ChildProcess::new(child, reader, matches!(self.err, OutDest::Pipe), head)?; - - Ok(PipelineData::ByteStream( - ByteStream::child(child, head), - None, - )) - } - } +/// Evaluates an expression, coercing the values to strings. +/// +/// Note: The parser currently has a special hack that retains surrounding +/// quotes for string literals in `Expression`, so that we can decide whether +/// the expression is considered a bare string. The hack doesn't affect string +/// literals within lists or records. This function will remove the quotes +/// before evaluating the expression. +fn eval_argument( + engine_state: &EngineState, + stack: &mut Stack, + expr: &Expression, + spread: bool, +) -> Result, ShellError> { + // Remove quotes from string literals. + let mut expr = expr.clone(); + if let Expr::String(s) = &expr.expr { + expr.expr = Expr::String(remove_quotes(s).into()); } - pub fn create_process( - &self, - stdin: Stdio, - use_cmd: bool, - span: Span, - ) -> Result<(CommandSys, Option), ShellError> { - let mut process = if let Some(d) = self.env_vars.get("PWD") { - let mut process = if use_cmd { - self.spawn_cmd_command(d) + let eval = get_eval_expression(engine_state); + match eval(engine_state, stack, &expr)? { + Value::List { vals, .. } => { + if spread { + vals.into_iter() + .map(|val| val.coerce_into_string()) + .collect() } else { - self.create_command(d)? - }; - - // do not try to set current directory if cwd does not exist - if Path::new(&d).exists() { - process.current_dir(d); + Err(ShellError::CannotPassListToExternal { + arg: String::from_utf8_lossy(engine_state.get_span_contents(expr.span)).into(), + span: expr.span, + }) } - process - } else { - return Err(ShellError::GenericError{ - error: "Current directory not found".into(), - msg: "did not find PWD environment variable".into(), - span: Some(span), - help: Some(concat!( - "The environment variable 'PWD' was not found. ", - "It is required to define the current directory when running an external command." - ).into()), - inner:Vec::new(), - }); - }; - - process.envs(&self.env_vars); - - // If the external is not the last command, its output will get piped - // either as a string or binary - let reader = if matches!(self.out, OutDest::Pipe) && matches!(self.err, OutDest::Pipe) { - let (reader, writer) = os_pipe::pipe()?; - let writer_clone = writer.try_clone()?; - process.stdout(writer); - process.stderr(writer_clone); - Some(reader) - } else { - process.stdout(Stdio::try_from(&self.out)?); - process.stderr(Stdio::try_from(&self.err)?); - None - }; - - process.stdin(stdin); - - Ok((process, reader)) - } - - fn create_command(&self, cwd: &str) -> Result { - // in all the other cases shell out - if cfg!(windows) { - //TODO. This should be modifiable from the config file. - // We could give the option to call from powershell - // for minimal builds cwd is unused - if self.name.item.ends_with(".cmd") || self.name.item.ends_with(".bat") { - Ok(self.spawn_cmd_command(cwd)) + } + value => { + if spread { + Err(ShellError::CannotSpreadAsList { span: expr.span }) } else { - self.spawn_simple_command(cwd) + Ok(vec![value.coerce_into_string()?]) } - } else { - self.spawn_simple_command(cwd) } } - - /// Spawn a command without shelling out to an external shell - pub fn spawn_simple_command(&self, cwd: &str) -> Result { - let (head, _, _) = trim_enclosing_quotes(&self.name.item); - let head = nu_path::expand_to_real_path(head) - .to_string_lossy() - .to_string(); - - let mut process = std::process::Command::new(head); - - for (arg, arg_keep_raw) in self.args.iter().zip(self.arg_keep_raw.iter()) { - trim_expand_and_apply_arg(&mut process, arg, arg_keep_raw, cwd); - } - - Ok(process) - } - - /// Spawn a cmd command with `cmd /c args...` - pub fn spawn_cmd_command(&self, cwd: &str) -> std::process::Command { - let mut process = std::process::Command::new("cmd"); - - // Disable AutoRun - // TODO: There should be a config option to enable/disable this - // Alternatively (even better) a config option to specify all the arguments to pass to cmd - process.arg("/D"); - - process.arg("/c"); - process.arg(&self.name.item); - for (arg, arg_keep_raw) in self.args.iter().zip(self.arg_keep_raw.iter()) { - // https://stackoverflow.com/questions/1200235/how-to-pass-a-quoted-pipe-character-to-cmd-exe - // cmd.exe needs to have a caret to escape a pipe - let arg = Spanned { - item: arg.item.replace('|', "^|"), - span: arg.span, - }; - - trim_expand_and_apply_arg(&mut process, &arg, arg_keep_raw, cwd) - } - - process - } } -fn trim_expand_and_apply_arg( - process: &mut CommandSys, - arg: &Spanned, - arg_keep_raw: &bool, - cwd: &str, -) { - // if arg is quoted, like "aa", 'aa', `aa`, or: - // if arg is a variable or String interpolation, like: $variable_name, $"($variable_name)" - // `as_a_whole` will be true, so nu won't remove the inner quotes. - let (trimmed_args, mut run_glob_expansion, mut keep_raw) = trim_enclosing_quotes(&arg.item); - if *arg_keep_raw { - keep_raw = true; - // it's a list or a variable, don't run glob expansion either - run_glob_expansion = false; - } - let mut arg = Spanned { - item: if keep_raw { - trimmed_args - } else { - remove_quotes(trimmed_args) - }, - span: arg.span, +/// Returns whether an expression is considered a bare string. +/// +/// Bare strings are defined as string literals that are either unquoted or +/// quoted by backticks. Raw strings or string interpolations don't count. +fn is_bare_string(expr: &Expression) -> bool { + let Expr::String(s) = &expr.expr else { + return false; }; - if !keep_raw { - arg.item = nu_path::expand_tilde(arg.item) - .to_string_lossy() - .to_string(); - } - let cwd = PathBuf::from(cwd); - if arg.item.contains('*') && run_glob_expansion { - // we need to run glob expansion, so it's NeedExpand. - let path = Spanned { - item: NuGlob::Expand(arg.item.clone()), - span: arg.span, - }; - if let Ok((prefix, matches)) = nu_engine::glob_from(&path, &cwd, arg.span, None) { - let matches: Vec<_> = matches.collect(); - - // FIXME: do we want to special-case this further? We might accidentally expand when they don't - // intend to - if matches.is_empty() { - process.arg(&arg.item); - } - for m in matches { - if let Ok(arg) = m { - let arg = if let Some(prefix) = &prefix { - if let Ok(remainder) = arg.strip_prefix(prefix) { - let new_prefix = if let Some(pfx) = diff_paths(prefix, &cwd) { - pfx - } else { - prefix.to_path_buf() - }; - - new_prefix.join(remainder).to_string_lossy().to_string() - } else { - arg.to_string_lossy().to_string() - } - } else { - arg.to_string_lossy().to_string() - }; - - process.arg(&arg); - } else { - process.arg(&arg.item); - } - } - } - } else { - process.arg(&arg.item); - } + let quoted_by_double_quotes = s.len() >= 2 && s.starts_with('"') && s.ends_with('"'); + let quoted_by_single_quotes = s.len() >= 2 && s.starts_with('\'') && s.ends_with('\''); + !quoted_by_double_quotes && !quoted_by_single_quotes } -/// Given an invalid command name, try to suggest an alternative -fn suggest_command(attempted_command: &str, engine_state: &EngineState) -> Option { - let commands = engine_state.get_signatures(false); - let command_folded_case = attempted_command.to_folded_case(); - let search_term_match = commands.iter().find(|sig| { +/// Performs tilde expansion on `arg`. Returns the original string if `arg` +/// doesn't start with tilde. +fn expand_tilde(arg: &str) -> String { + nu_path::expand_tilde(arg).to_string_lossy().to_string() +} + +/// Performs glob expansion on `arg`. If the expansion found no matches or the pattern +/// is not a valid glob, then this returns the original string as the expansion result. +/// +/// Note: This matches the default behavior of Bash, but is known to be +/// error-prone. We might want to change this behavior in the future. +fn expand_glob(arg: &str, cwd: &Path, span: Span) -> Result, ShellError> { + let Ok(paths) = nu_glob::glob_with_parent(arg, nu_glob::MatchOptions::default(), cwd) else { + return Ok(vec![arg.into()]); + }; + + let mut result = vec![]; + for path in paths { + let path = path.map_err(|err| ShellError::IOErrorSpanned { + msg: format!("{}: {:?}", err.path().display(), err.error()), + span, + })?; + // Strip PWD from the resulting paths if possible. + let path_stripped = if let Ok(remainder) = path.strip_prefix(cwd) { + // If stripping PWD results in an empty path, return `.` instead. + if remainder.components().next().is_none() { + Path::new(".") + } else { + remainder + } + } else { + &path + }; + let path_string = path_stripped.to_string_lossy().to_string(); + result.push(path_string); + } + + if result.is_empty() { + result.push(arg.to_string()); + } + + Ok(result) +} + +/// Transforms `--option="value"` into `--option=value`. `value` can be quoted +/// with double quotes, single quotes, or backticks. Only removes the outermost +/// pair of quotes after the equal sign. +fn remove_inner_quotes(arg: &str) -> Cow<'_, str> { + // Check that `arg` is a long option. + if !arg.starts_with("--") { + return Cow::Borrowed(arg); + } + // Split `arg` on the first `=`. + let Some((option, value)) = arg.split_once('=') else { + return Cow::Borrowed(arg); + }; + // Check that `option` doesn't contain quotes. + if option.contains('"') || option.contains('\'') || option.contains('`') { + return Cow::Borrowed(arg); + } + // Remove the outermost pair of quotes from `value`. + let value = remove_quotes(value); + Cow::Owned(format!("{option}={value}")) +} + +/// Write `PipelineData` into `writer`. If `PipelineData` is not binary, it is +/// first rendered using the `table` command. +/// +/// Note: Avoid using this function when piping data from an external command to +/// another external command, because it copies data unnecessarily. Instead, +/// extract the pipe from the `PipelineData::ByteStream` of the first command +/// and hand it to the second command directly. +fn write_pipeline_data( + mut engine_state: EngineState, + mut stack: Stack, + data: PipelineData, + mut writer: impl Write, +) -> Result<(), ShellError> { + if let PipelineData::ByteStream(stream, ..) = data { + stream.write_to(writer)?; + } else if let PipelineData::Value(Value::Binary { val, .. }, ..) = data { + writer.write_all(&val)?; + } else { + stack.start_capture(); + + // Turn off color as we pass data through + Arc::make_mut(&mut engine_state.config).use_ansi_coloring = false; + + // Invoke the `table` command. + let output = + crate::Table.run(&engine_state, &mut stack, &Call::new(Span::unknown()), data)?; + + // Write the output. + for value in output { + let bytes = value.coerce_into_binary()?; + writer.write_all(&bytes)?; + } + } + Ok(()) +} + +/// Returns a helpful error message given an invalid command name, +pub fn command_not_found( + name: &str, + span: Span, + engine_state: &EngineState, + stack: &mut Stack, +) -> ShellError { + // Run the `command_not_found` hook if there is one. + if let Some(hook) = &engine_state.config.hooks.command_not_found { + let mut stack = stack.start_capture(); + // Set a special environment variable to avoid infinite loops when the + // `command_not_found` hook triggers itself. + let canary = "ENTERED_COMMAND_NOT_FOUND"; + if stack.has_env_var(engine_state, canary) { + return ShellError::ExternalCommand { + label: format!( + "Command {name} not found while running the `command_not_found` hook" + ), + help: "Make sure the `command_not_found` hook itself does not use unknown commands" + .into(), + span, + }; + } + stack.add_env_var(canary.into(), Value::bool(true, Span::unknown())); + + let output = eval_hook( + &mut engine_state.clone(), + &mut stack, + None, + vec![("cmd_name".into(), Value::string(name, span))], + hook, + "command_not_found", + ); + + // Remove the special environment variable that we just set. + stack.remove_env_var(engine_state, canary); + + match output { + Ok(PipelineData::Value(Value::String { val, .. }, ..)) => { + return ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: val, + span, + }; + } + Err(err) => { + return err; + } + _ => { + // The hook did not return a string, so ignore it. + } + } + } + + // If the name is one of the removed commands, recommend a replacement. + if let Some(replacement) = crate::removed_commands().get(&name.to_lowercase()) { + return ShellError::RemovedCommand { + removed: name.to_lowercase(), + replacement: replacement.clone(), + span, + }; + } + + // The command might be from another module. Try to find it. + if let Some(module) = engine_state.which_module_has_decl(name.as_bytes(), &[]) { + let module = String::from_utf8_lossy(module); + // Is the command already imported? + let full_name = format!("{module} {name}"); + if engine_state.find_decl(full_name.as_bytes(), &[]).is_some() { + return ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: format!("Did you mean `{full_name}`?"), + span, + }; + } else { + return ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: format!("A command with that name exists in module `{module}`. Try importing it with `use`"), + span, + }; + } + } + + // Try to match the name with the search terms of existing commands. + let signatures = engine_state.get_signatures(false); + if let Some(sig) = signatures.iter().find(|sig| { sig.search_terms .iter() - .any(|term| term.to_folded_case() == command_folded_case) - }); - match search_term_match { - Some(sig) => Some(sig.name.clone()), - None => { - let command_names: Vec = commands.iter().map(|sig| sig.name.clone()).collect(); - did_you_mean(&command_names, attempted_command) + .any(|term| term.to_folded_case() == name.to_folded_case()) + }) { + return ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: format!("Did you mean `{}`?", sig.name), + span, + }; + } + + // Try a fuzzy search on the names of all existing commands. + if let Some(cmd) = did_you_mean(signatures.iter().map(|sig| &sig.name), name) { + // The user is invoking an external command with the same name as a + // built-in command. Remind them of this. + if cmd == name { + return ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: "There is a built-in command with the same name".into(), + span, + }; } + return ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: format!("Did you mean `{cmd}`?"), + span, + }; + } + + // We found nothing useful. Give up and return a generic error message. + ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: format!("`{name}` is neither a Nushell built-in or a known external command"), + span, } } -/// This function returns a tuple with 3 items: -/// 1st item: trimmed string. -/// 2nd item: a boolean value indicate if it's ok to run glob expansion. -/// 3rd item: a boolean value indicate if we need to keep raw string. -fn trim_enclosing_quotes(input: &str) -> (String, bool, bool) { - let mut chars = input.chars(); - - match (chars.next(), chars.next_back()) { - (Some('"'), Some('"')) => (chars.collect(), false, true), - (Some('\''), Some('\'')) => (chars.collect(), false, true), - // We treat back-quoted strings as bare words, so there's no need to keep them as raw strings - (Some('`'), Some('`')) => (chars.collect(), true, false), - _ => (input.to_string(), true, false), - } +/// Searches for the absolute path of an executable by name. `.bat` and `.cmd` +/// files are recognized as executables on Windows. +/// +/// This is a wrapper around `which::which_in()` except that, on Windows, it +/// also searches the current directory before any PATH entries. +/// +/// Note: the `which.rs` crate always uses PATHEXT from the environment. As +/// such, changing PATHEXT within Nushell doesn't work without updating the +/// actual environment of the Nushell process. +pub fn which(name: &str, paths: &str, cwd: &Path) -> Option { + #[cfg(windows)] + let paths = format!("{};{}", cwd.display(), paths); + which::which_in(name, Some(paths), cwd).ok() } -fn remove_quotes(input: String) -> String { - let mut chars = input.chars(); - - match (chars.next_back(), input.contains('=')) { - (Some('"'), true) => chars - .collect::() - .replacen('"', "", 1) - .replace(r#"\""#, "\""), - (Some('\''), true) => chars.collect::().replacen('\'', "", 1), - _ => input, - } +/// Returns true if `name` is a (somewhat useful) CMD internal command. The full +/// list can be found at https://ss64.com/nt/syntax-internal.html +fn is_cmd_internal_command(name: &str) -> bool { + const COMMANDS: &[&str] = &[ + "ASSOC", "CLS", "ECHO", "FTYPE", "MKLINK", "PAUSE", "START", "VER", "VOL", + ]; + COMMANDS.iter().any(|cmd| cmd.eq_ignore_ascii_case(name)) } -fn extract_stdio(pipeline: PipelineData) -> (PipelineData, Stdio) { - match pipeline { - PipelineData::ByteStream(stream, metadata) => match stream.into_stdio() { - Ok(pipe) => (PipelineData::Empty, pipe), - Err(stream) => (PipelineData::ByteStream(stream, metadata), Stdio::piped()), - }, - PipelineData::Empty => (PipelineData::Empty, Stdio::inherit()), - data => (data, Stdio::piped()), +/// Returns true if a string contains CMD special characters. +#[cfg(windows)] +fn has_cmd_special_character(s: &str) -> bool { + const SPECIAL_CHARS: &[char] = &['<', '>', '&', '|', '^']; + SPECIAL_CHARS.iter().any(|c| s.contains(*c)) +} + +/// Escape an argument for CMD internal commands. The result can be safely +/// passed to `raw_arg()`. +#[cfg(windows)] +fn escape_cmd_argument(arg: &Spanned) -> Result, ShellError> { + let Spanned { item: arg, span } = arg; + if arg.contains('"') { + // If `arg` is already quoted by double quotes, confirm there's no + // embedded double quotes, then leave it as is. + if arg.chars().filter(|c| *c == '"').count() == 2 + && arg.starts_with('"') + && arg.ends_with('"') + { + Ok(Cow::Borrowed(arg)) + } else { + Err(ShellError::ExternalCommand { + label: "Arguments to CMD internal commands cannot contain embedded double quotes" + .into(), + help: "CMD doesn't support escaping double quotes inside double quotes".into(), + span: *span, + }) + } + } else if arg.contains(' ') || has_cmd_special_character(arg) { + // If `arg` contains space or special characters, quote the entire argument by double quotes. + Ok(Cow::Owned(format!("\"{arg}\""))) + } else { + Ok(Cow::Borrowed(arg)) } } #[cfg(test)] mod test { use super::*; + use nu_protocol::ast::ListItem; #[test] - fn remove_quotes_argument_with_equal_test() { - let input = r#"--file="my_file.txt""#.into(); - let res = remove_quotes(input); - - assert_eq!("--file=my_file.txt", res) + fn test_remove_quotes() { + assert_eq!(remove_quotes(r#""#), r#""#); + assert_eq!(remove_quotes(r#"'"#), r#"'"#); + assert_eq!(remove_quotes(r#"''"#), r#""#); + assert_eq!(remove_quotes(r#""foo""#), r#"foo"#); + assert_eq!(remove_quotes(r#"`foo '"' bar`"#), r#"foo '"' bar"#); + assert_eq!(remove_quotes(r#"'foo' bar"#), r#"'foo' bar"#); + assert_eq!(remove_quotes(r#"r#'foo'#"#), r#"r#'foo'#"#); } #[test] - fn argument_without_equal_test() { - let input = r#"--file "my_file.txt""#.into(); - let res = remove_quotes(input); + fn test_eval_argument() { + fn expression(expr: Expr) -> Expression { + Expression { + expr, + span: Span::unknown(), + ty: Type::Any, + custom_completion: None, + } + } - assert_eq!(r#"--file "my_file.txt""#, res) + fn eval(expr: Expr, spread: bool) -> Result, ShellError> { + let engine_state = EngineState::new(); + let mut stack = Stack::new(); + eval_argument(&engine_state, &mut stack, &expression(expr), spread) + } + + let actual = eval(Expr::String("".into()), false).unwrap(); + let expected = &[""]; + assert_eq!(actual, expected); + + let actual = eval(Expr::String("'foo'".into()), false).unwrap(); + let expected = &["foo"]; + assert_eq!(actual, expected); + + let actual = eval(Expr::RawString("'foo'".into()), false).unwrap(); + let expected = &["'foo'"]; + assert_eq!(actual, expected); + + let actual = eval(Expr::List(vec![]), true).unwrap(); + let expected: &[&str] = &[]; + assert_eq!(actual, expected); + + let actual = eval( + Expr::List(vec![ + ListItem::Item(expression(Expr::String("'foo'".into()))), + ListItem::Item(expression(Expr::String("bar".into()))), + ]), + true, + ) + .unwrap(); + let expected = &["'foo'", "bar"]; + assert_eq!(actual, expected); + + eval(Expr::String("".into()), true).unwrap_err(); + eval(Expr::List(vec![]), false).unwrap_err(); } #[test] - fn remove_quotes_argument_with_single_quotes_test() { - let input = r#"--file='my_file.txt'"#.into(); - let res = remove_quotes(input); + fn test_expand_glob() { + let tempdir = tempfile::tempdir().unwrap(); + let cwd = tempdir.path(); + std::fs::File::create(cwd.join("a.txt")).unwrap(); + std::fs::File::create(cwd.join("b.txt")).unwrap(); - assert_eq!("--file=my_file.txt", res) + let actual = expand_glob("*.txt", cwd, Span::unknown()).unwrap(); + let expected = &["a.txt", "b.txt"]; + assert_eq!(actual, expected); + + let actual = expand_glob("'*.txt'", cwd, Span::unknown()).unwrap(); + let expected = &["'*.txt'"]; + assert_eq!(actual, expected); + + let actual = expand_glob(cwd.to_str().unwrap(), cwd, Span::unknown()).unwrap(); + let expected = &["."]; + assert_eq!(actual, expected); + + let actual = expand_glob("[*.txt", cwd, Span::unknown()).unwrap(); + let expected = &["[*.txt"]; + assert_eq!(actual, expected); } #[test] - fn argument_with_inner_quotes_test() { - let input = r#"sh -c 'echo a'"#.into(); - let res = remove_quotes(input); + fn test_remove_inner_quotes() { + let actual = remove_inner_quotes(r#"--option=value"#); + let expected = r#"--option=value"#; + assert_eq!(actual, expected); - assert_eq!("sh -c 'echo a'", res) + let actual = remove_inner_quotes(r#"--option="value""#); + let expected = r#"--option=value"#; + assert_eq!(actual, expected); + + let actual = remove_inner_quotes(r#"--option='value'"#); + let expected = r#"--option=value"#; + assert_eq!(actual, expected); + + let actual = remove_inner_quotes(r#"--option "value""#); + let expected = r#"--option "value""#; + assert_eq!(actual, expected); + } + + #[test] + fn test_write_pipeline_data() { + let engine_state = EngineState::new(); + let stack = Stack::new(); + + let mut buf = vec![]; + let input = PipelineData::Empty; + write_pipeline_data(engine_state.clone(), stack.clone(), input, &mut buf).unwrap(); + assert_eq!(buf, b""); + + let mut buf = vec![]; + let input = PipelineData::Value(Value::string("foo", Span::unknown()), None); + write_pipeline_data(engine_state.clone(), stack.clone(), input, &mut buf).unwrap(); + assert_eq!(buf, b"foo"); + + let mut buf = vec![]; + let input = PipelineData::Value(Value::binary(b"foo", Span::unknown()), None); + write_pipeline_data(engine_state.clone(), stack.clone(), input, &mut buf).unwrap(); + assert_eq!(buf, b"foo"); + + let mut buf = vec![]; + let input = PipelineData::ByteStream( + ByteStream::read( + b"foo".as_slice(), + Span::unknown(), + None, + ByteStreamType::Unknown, + ), + None, + ); + write_pipeline_data(engine_state.clone(), stack.clone(), input, &mut buf).unwrap(); + assert_eq!(buf, b"foo"); } } diff --git a/crates/nu-command/src/system/which_.rs b/crates/nu-command/src/system/which_.rs index 1244a57d99..81b6057864 100644 --- a/crates/nu-command/src/system/which_.rs +++ b/crates/nu-command/src/system/which_.rs @@ -1,5 +1,5 @@ -use log::trace; use nu_engine::{command_prelude::*, env}; +use nu_protocol::engine::CommandType; use std::{ffi::OsStr, path::Path}; #[derive(Clone)] @@ -51,14 +51,14 @@ impl Command for Which { fn entry( arg: impl Into, path: impl Into, - cmd_type: impl Into, + cmd_type: CommandType, span: Span, ) -> Value { Value::record( record! { - "command" => Value::string(arg.into(), span), - "path" => Value::string(path.into(), span), - "type" => Value::string(cmd_type.into(), span), + "command" => Value::string(arg, span), + "path" => Value::string(path, span), + "type" => Value::string(cmd_type.to_string(), span), }, span, ) @@ -66,17 +66,8 @@ fn entry( fn get_entry_in_commands(engine_state: &EngineState, name: &str, span: Span) -> Option { if let Some(decl_id) = engine_state.find_decl(name.as_bytes(), &[]) { - let cmd_type = if engine_state.get_decl(decl_id).is_custom_command() { - "custom" - } else if engine_state.get_decl(decl_id).is_alias() { - "alias" - } else { - "built-in" - }; - - trace!("Found command: {}", name); - - Some(entry(name, "", cmd_type, span)) + let decl = engine_state.get_decl(decl_id); + Some(entry(name, "", decl.command_type(), span)) } else { None } @@ -109,7 +100,7 @@ fn get_first_entry_in_path( paths: impl AsRef, ) -> Option { which::which_in(item, Some(paths), cwd) - .map(|path| entry(item, path.to_string_lossy().to_string(), "external", span)) + .map(|path| entry(item, path.to_string_lossy(), CommandType::External, span)) .ok() } @@ -132,7 +123,7 @@ fn get_all_entries_in_path( ) -> Vec { which::which_in_all(&item, Some(paths), cwd) .map(|iter| { - iter.map(|path| entry(item, path.to_string_lossy().to_string(), "external", span)) + iter.map(|path| entry(item, path.to_string_lossy(), CommandType::External, span)) .collect() }) .unwrap_or_default() diff --git a/crates/nu-command/src/viewers/griddle.rs b/crates/nu-command/src/viewers/griddle.rs index e0856e0a70..8b39bf5649 100644 --- a/crates/nu-command/src/viewers/griddle.rs +++ b/crates/nu-command/src/viewers/griddle.rs @@ -5,6 +5,7 @@ use nu_engine::{command_prelude::*, env_to_string}; use nu_protocol::Config; use nu_term_grid::grid::{Alignment, Cell, Direction, Filling, Grid, GridOptions}; use nu_utils::get_ls_colors; +use std::path::Path; use terminal_size::{Height, Width}; #[derive(Clone)] @@ -67,6 +68,7 @@ prints out the list properly."# }; let use_grid_icons = config.use_grid_icons; let use_color: bool = color_param && config.use_ansi_coloring; + let cwd = engine_state.cwd(Some(stack))?; match input { PipelineData::Value(Value::List { vals, .. }, ..) => { @@ -81,6 +83,7 @@ prints out the list properly."# separator_param, env_str, use_grid_icons, + &cwd, )?) } else { Ok(PipelineData::empty()) @@ -98,6 +101,7 @@ prints out the list properly."# separator_param, env_str, use_grid_icons, + &cwd, )?) } else { // dbg!(data); @@ -120,6 +124,7 @@ prints out the list properly."# separator_param, env_str, use_grid_icons, + &cwd, )?) } x => { @@ -161,6 +166,7 @@ prints out the list properly."# } } +#[allow(clippy::too_many_arguments)] fn create_grid_output( items: Vec<(usize, String, String)>, call: &Call, @@ -169,6 +175,7 @@ fn create_grid_output( separator_param: Option, env_str: Option, use_grid_icons: bool, + cwd: &Path, ) -> Result { let ls_colors = get_ls_colors(env_str); @@ -196,8 +203,8 @@ fn create_grid_output( if use_color { if use_grid_icons { let no_ansi = nu_utils::strip_ansi_unlikely(&value); - let path = std::path::Path::new(no_ansi.as_ref()); - let icon = icon_for_file(path, call.head)?; + let path = cwd.join(no_ansi.as_ref()); + let icon = icon_for_file(&path, call.head)?; let ls_colors_style = ls_colors.style_for_path(path); let icon_style = match ls_colors_style { diff --git a/crates/nu-command/src/viewers/table.rs b/crates/nu-command/src/viewers/table.rs index 26b8c921c5..2fe9319821 100644 --- a/crates/nu-command/src/viewers/table.rs +++ b/crates/nu-command/src/viewers/table.rs @@ -5,6 +5,7 @@ use lscolors::{LsColors, Style}; use nu_color_config::{color_from_hex, StyleComputer, TextStyle}; use nu_engine::{command_prelude::*, env::get_config, env_to_string}; +use nu_pretty_hex::HexConfig; use nu_protocol::{ ByteStream, Config, DataSource, ListStream, PipelineMetadata, TableMode, ValueIterator, }; @@ -15,7 +16,7 @@ use nu_table::{ use nu_utils::get_ls_colors; use std::{ collections::VecDeque, - io::{Cursor, IsTerminal}, + io::{IsTerminal, Read}, path::PathBuf, str::FromStr, sync::{atomic::AtomicBool, Arc}, @@ -364,16 +365,18 @@ fn handle_table_command( ) -> Result { let span = input.data.span().unwrap_or(input.call.head); match input.data { + // Binary streams should behave as if they really are `binary` data, and printed as hex + PipelineData::ByteStream(stream, _) if stream.type_() == ByteStreamType::Binary => Ok( + PipelineData::ByteStream(pretty_hex_stream(stream, input.call.head), None), + ), PipelineData::ByteStream(..) => Ok(input.data), PipelineData::Value(Value::Binary { val, .. }, ..) => { - let bytes = { - let mut str = nu_pretty_hex::pretty_hex(&val); - str.push('\n'); - str.into_bytes() - }; let ctrlc = input.engine_state.ctrlc.clone(); - let stream = ByteStream::read(Cursor::new(bytes), input.call.head, ctrlc); - Ok(PipelineData::ByteStream(stream, None)) + let stream = ByteStream::read_binary(val, input.call.head, ctrlc); + Ok(PipelineData::ByteStream( + pretty_hex_stream(stream, input.call.head), + None, + )) } // None of these two receive a StyleComputer because handle_row_stream() can produce it by itself using engine_state and stack. PipelineData::Value(Value::List { vals, .. }, metadata) => { @@ -410,6 +413,70 @@ fn handle_table_command( } } +fn pretty_hex_stream(stream: ByteStream, span: Span) -> ByteStream { + let mut cfg = HexConfig { + // We are going to render the title manually first + title: true, + // If building on 32-bit, the stream size might be bigger than a usize + length: stream.known_size().and_then(|sz| sz.try_into().ok()), + ..HexConfig::default() + }; + + // This won't really work for us + debug_assert!(cfg.width > 0, "the default hex config width was zero"); + + let mut read_buf = Vec::with_capacity(cfg.width); + + let mut reader = if let Some(reader) = stream.reader() { + reader + } else { + // No stream to read from + return ByteStream::read_string("".into(), span, None); + }; + + ByteStream::from_fn(span, None, ByteStreamType::String, move |buffer| { + // Turn the buffer into a String we can write to + let mut write_buf = std::mem::take(buffer); + write_buf.clear(); + // SAFETY: we just truncated it empty + let mut write_buf = unsafe { String::from_utf8_unchecked(write_buf) }; + + // Write the title at the beginning + if cfg.title { + nu_pretty_hex::write_title(&mut write_buf, cfg, true).expect("format error"); + cfg.title = false; + + // Put the write_buf back into buffer + *buffer = write_buf.into_bytes(); + + Ok(true) + } else { + // Read up to `cfg.width` bytes + read_buf.clear(); + (&mut reader) + .take(cfg.width as u64) + .read_to_end(&mut read_buf) + .err_span(span)?; + + if !read_buf.is_empty() { + nu_pretty_hex::hex_write(&mut write_buf, &read_buf, cfg, Some(true)) + .expect("format error"); + write_buf.push('\n'); + + // Advance the address offset for next time + cfg.address_offset += read_buf.len(); + + // Put the write_buf back into buffer + *buffer = write_buf.into_bytes(); + + Ok(true) + } else { + Ok(false) + } + } + }) +} + fn handle_record( input: CmdInput, cfg: TableConfig, @@ -608,7 +675,8 @@ fn handle_row_stream( ctrlc.clone(), cfg, ); - let stream = ByteStream::from_result_iter(paginator, input.call.head, None); + let stream = + ByteStream::from_result_iter(paginator, input.call.head, None, ByteStreamType::String); Ok(PipelineData::ByteStream(stream, None)) } diff --git a/crates/nu-command/tests/commands/all.rs b/crates/nu-command/tests/commands/all.rs index bcf6f86a45..8ed5a6f34f 100644 --- a/crates/nu-command/tests/commands/all.rs +++ b/crates/nu-command/tests/commands/all.rs @@ -55,7 +55,9 @@ fn checks_if_all_returns_error_with_invalid_command() { "# )); - assert!(actual.err.contains("can't run executable") || actual.err.contains("did you mean")); + assert!( + actual.err.contains("Command `st` not found") && actual.err.contains("Did you mean `ast`?") + ); } #[test] diff --git a/crates/nu-command/tests/commands/any.rs b/crates/nu-command/tests/commands/any.rs index 0e43f26cfd..f8af25eb61 100644 --- a/crates/nu-command/tests/commands/any.rs +++ b/crates/nu-command/tests/commands/any.rs @@ -41,7 +41,9 @@ fn checks_if_any_returns_error_with_invalid_command() { "# )); - assert!(actual.err.contains("can't run executable") || actual.err.contains("did you mean")); + assert!( + actual.err.contains("Command `st` not found") && actual.err.contains("Did you mean `ast`?") + ); } #[test] diff --git a/crates/nu-command/tests/commands/bytes/collect.rs b/crates/nu-command/tests/commands/bytes/collect.rs new file mode 100644 index 0000000000..768ab16df4 --- /dev/null +++ b/crates/nu-command/tests/commands/bytes/collect.rs @@ -0,0 +1,27 @@ +use nu_test_support::{nu, pipeline}; + +#[test] +fn test_stream() { + let actual = nu!(pipeline( + " + [0x[01] 0x[02] 0x[03] 0x[04]] + | filter {true} + | bytes collect 0x[aa aa] + | encode hex + " + )); + assert_eq!(actual.out, "01AAAA02AAAA03AAAA04"); +} + +#[test] +fn test_stream_type() { + let actual = nu!(pipeline( + " + [0x[01] 0x[02] 0x[03] 0x[04]] + | filter {true} + | bytes collect 0x[00] + | describe -n + " + )); + assert_eq!(actual.out, "binary (stream)"); +} diff --git a/crates/nu-command/tests/commands/bytes/mod.rs b/crates/nu-command/tests/commands/bytes/mod.rs new file mode 100644 index 0000000000..10b2a494f8 --- /dev/null +++ b/crates/nu-command/tests/commands/bytes/mod.rs @@ -0,0 +1 @@ +mod collect; diff --git a/crates/nu-command/tests/commands/complete.rs b/crates/nu-command/tests/commands/complete.rs index a5a8b6a256..329e95ad71 100644 --- a/crates/nu-command/tests/commands/complete.rs +++ b/crates/nu-command/tests/commands/complete.rs @@ -24,7 +24,7 @@ fn basic_exit_code() { #[test] fn error() { let actual = nu!("not-found | complete"); - assert!(actual.err.contains("executable was not found")); + assert!(actual.err.contains("Command `not-found` not found")); } #[test] diff --git a/crates/nu-command/tests/commands/detect_columns.rs b/crates/nu-command/tests/commands/detect_columns.rs index d605eb4aa1..5bc057fa18 100644 --- a/crates/nu-command/tests/commands/detect_columns.rs +++ b/crates/nu-command/tests/commands/detect_columns.rs @@ -31,12 +31,12 @@ fn detect_columns_with_legacy_and_flag_c() { ( "$\"c1 c2 c3 c4 c5(char nl)a b c d e\"", "[[c1,c3,c4,c5]; ['a b',c,d,e]]", - "0..1", + "0..0", ), ( "$\"c1 c2 c3 c4 c5(char nl)a b c d e\"", "[[c1,c2,c3,c4]; [a,b,c,'d e']]", - "(-2)..(-1)", + "(-2)..(-2)", ), ( "$\"c1 c2 c3 c4 c5(char nl)a b c d e\"", @@ -77,7 +77,7 @@ drwxr-xr-x 4 root root 4.0K Mar 20 08:18 ~(char nl) ['drwxr-xr-x', '4', 'root', 'root', '4.0K', 'Mar 20 08:18', '~'], ['-rw-r--r--', '1', 'root', 'root', '3.0K', 'Mar 20 07:23', '~asdf'] ]"; - let range = "5..7"; + let range = "5..6"; let cmd = format!( "({} | detect columns -c {} -s 1 --no-headers) == {}", pipeline(body), diff --git a/crates/nu-command/tests/commands/first.rs b/crates/nu-command/tests/commands/first.rs index e01478f820..23ccda6669 100644 --- a/crates/nu-command/tests/commands/first.rs +++ b/crates/nu-command/tests/commands/first.rs @@ -68,6 +68,20 @@ fn gets_first_byte() { assert_eq!(actual.out, "170"); } +#[test] +fn gets_first_bytes_from_stream() { + let actual = nu!("(1.. | each { 0x[aa bb cc] } | bytes collect | first 2) == 0x[aa bb]"); + + assert_eq!(actual.out, "true"); +} + +#[test] +fn gets_first_byte_from_stream() { + let actual = nu!("1.. | each { 0x[aa bb cc] } | bytes collect | first"); + + assert_eq!(actual.out, "170"); +} + #[test] // covers a situation where `first` used to behave strangely on list input fn works_with_binary_list() { diff --git a/crates/nu-command/tests/commands/last.rs b/crates/nu-command/tests/commands/last.rs index b0c67e49be..986b433ea7 100644 --- a/crates/nu-command/tests/commands/last.rs +++ b/crates/nu-command/tests/commands/last.rs @@ -68,6 +68,20 @@ fn gets_last_byte() { assert_eq!(actual.out, "204"); } +#[test] +fn gets_last_bytes_from_stream() { + let actual = nu!("(1..10 | each { 0x[aa bb cc] } | bytes collect | last 2) == 0x[bb cc]"); + + assert_eq!(actual.out, "true"); +} + +#[test] +fn gets_last_byte_from_stream() { + let actual = nu!("1..10 | each { 0x[aa bb cc] } | bytes collect | last"); + + assert_eq!(actual.out, "204"); +} + #[test] fn last_errors_on_negative_index() { let actual = nu!("[1, 2, 3] | last -2"); diff --git a/crates/nu-command/tests/commands/mod.rs b/crates/nu-command/tests/commands/mod.rs index d7215e002b..922e804405 100644 --- a/crates/nu-command/tests/commands/mod.rs +++ b/crates/nu-command/tests/commands/mod.rs @@ -4,6 +4,7 @@ mod any; mod append; mod assignment; mod break_; +mod bytes; mod cal; mod cd; mod compact; diff --git a/crates/nu-command/tests/commands/open.rs b/crates/nu-command/tests/commands/open.rs index f857718c72..ac5e2f99e5 100644 --- a/crates/nu-command/tests/commands/open.rs +++ b/crates/nu-command/tests/commands/open.rs @@ -238,22 +238,6 @@ fn parses_xml() { assert_eq!(actual.out, "https://www.jntrnr.com/off-to-new-adventures/") } -#[cfg(feature = "dataframe")] -#[test] -fn parses_arrow_ipc() { - let actual = nu!( - cwd: "tests/fixtures/formats", pipeline( - " - dfr open caco3_plastics.arrow - | dfr into-nu - | first - | get origin - " - )); - - assert_eq!(actual.out, "SPAIN") -} - #[test] fn errors_if_file_not_found() { let actual = nu!( diff --git a/crates/nu-command/tests/commands/path/type_.rs b/crates/nu-command/tests/commands/path/type_.rs index 2317cea060..e67b957e4c 100644 --- a/crates/nu-command/tests/commands/path/type_.rs +++ b/crates/nu-command/tests/commands/path/type_.rs @@ -74,3 +74,14 @@ fn returns_type_of_existing_file_const() { assert_eq!(actual.out, "dir"); }) } + +#[test] +fn respects_cwd() { + Playground::setup("path_type_respects_cwd", |dirs, sandbox| { + sandbox.within("foo").with_files(&[EmptyFile("bar.txt")]); + + let actual = nu!(cwd: dirs.test(), "cd foo; 'bar.txt' | path type"); + + assert_eq!(actual.out, "file"); + }) +} diff --git a/crates/nu-command/tests/commands/skip/skip_.rs b/crates/nu-command/tests/commands/skip/skip_.rs index 790c58db4e..c98de496c8 100644 --- a/crates/nu-command/tests/commands/skip/skip_.rs +++ b/crates/nu-command/tests/commands/skip/skip_.rs @@ -1,13 +1,17 @@ use nu_test_support::nu; #[test] -fn binary_skip_will_raise_error() { - let actual = nu!( - cwd: "tests/fixtures/formats", - "open sample_data.ods --raw | skip 2" - ); +fn skips_bytes() { + let actual = nu!("(0x[aa bb cc] | skip 2) == 0x[cc]"); - assert!(actual.err.contains("only_supports_this_input_type")); + assert_eq!(actual.out, "true"); +} + +#[test] +fn skips_bytes_from_stream() { + let actual = nu!("([0 1] | each { 0x[aa bb cc] } | bytes collect | skip 2) == 0x[cc aa bb cc]"); + + assert_eq!(actual.out, "true"); } #[test] diff --git a/crates/nu-command/tests/commands/source_env.rs b/crates/nu-command/tests/commands/source_env.rs index 4e8da55c78..9dd0320a45 100644 --- a/crates/nu-command/tests/commands/source_env.rs +++ b/crates/nu-command/tests/commands/source_env.rs @@ -283,13 +283,17 @@ fn source_env_is_scoped() { let actual = nu!(cwd: dirs.test(), &inp.join("; ")); - assert!(actual.err.contains("executable was not found")); + assert!(actual + .err + .contains("Command `no-name-similar-to-this` not found")); let inp = &[r#"source-env spam.nu"#, r#"nor-similar-to-this"#]; let actual = nu!(cwd: dirs.test(), &inp.join("; ")); - assert!(actual.err.contains("executable was not found")); + assert!(actual + .err + .contains("Command `nor-similar-to-this` not found")); }) } diff --git a/crates/nu-command/tests/commands/str_/collect.rs b/crates/nu-command/tests/commands/str_/join.rs similarity index 65% rename from crates/nu-command/tests/commands/str_/collect.rs rename to crates/nu-command/tests/commands/str_/join.rs index 154ce30537..e04652e810 100644 --- a/crates/nu-command/tests/commands/str_/collect.rs +++ b/crates/nu-command/tests/commands/str_/join.rs @@ -22,6 +22,18 @@ fn test_2() { assert_eq!(actual.out, "abcd"); } +#[test] +fn test_stream() { + let actual = nu!("[a b c d] | filter {true} | str join ."); + assert_eq!(actual.out, "a.b.c.d"); +} + +#[test] +fn test_stream_type() { + let actual = nu!("[a b c d] | filter {true} | str join . | describe -n"); + assert_eq!(actual.out, "string (stream)"); +} + #[test] fn construct_a_path() { let actual = nu!(pipeline( diff --git a/crates/nu-command/tests/commands/str_/mod.rs b/crates/nu-command/tests/commands/str_/mod.rs index 9f1e90e853..43f59b3e10 100644 --- a/crates/nu-command/tests/commands/str_/mod.rs +++ b/crates/nu-command/tests/commands/str_/mod.rs @@ -1,5 +1,5 @@ -mod collect; mod into_string; +mod join; use nu_test_support::fs::Stub::FileWithContent; use nu_test_support::playground::Playground; @@ -269,7 +269,7 @@ fn substring_errors_if_start_index_is_greater_than_end_index() { cwd: dirs.test(), pipeline( r#" open sample.toml - | str substring 6..5 fortune.teller.phone + | str substring 6..4 fortune.teller.phone "# )); @@ -342,7 +342,7 @@ fn substrings_the_input_and_treats_start_index_as_zero_if_blank_start_index_give cwd: dirs.test(), pipeline( r#" open sample.toml - | str substring ..2 package.name + | str substring ..1 package.name | get package.name "# )); diff --git a/crates/nu-command/tests/commands/take/rows.rs b/crates/nu-command/tests/commands/take/rows.rs index d5f3d1c601..6c34b61310 100644 --- a/crates/nu-command/tests/commands/take/rows.rs +++ b/crates/nu-command/tests/commands/take/rows.rs @@ -35,6 +35,20 @@ fn fails_on_string() { assert!(actual.err.contains("command doesn't support")); } +#[test] +fn takes_bytes() { + let actual = nu!("(0x[aa bb cc] | take 2) == 0x[aa bb]"); + + assert_eq!(actual.out, "true"); +} + +#[test] +fn takes_bytes_from_stream() { + let actual = nu!("(1.. | each { 0x[aa bb cc] } | bytes collect | take 2) == 0x[aa bb]"); + + assert_eq!(actual.out, "true"); +} + #[test] // covers a situation where `take` used to behave strangely on list input fn works_with_binary_list() { diff --git a/crates/nu-command/tests/commands/touch.rs b/crates/nu-command/tests/commands/touch.rs index 26d0325729..5f454cdec4 100644 --- a/crates/nu-command/tests/commands/touch.rs +++ b/crates/nu-command/tests/commands/touch.rs @@ -515,3 +515,16 @@ fn respects_cwd() { assert!(path.exists()); }) } + +#[test] +fn reference_respects_cwd() { + Playground::setup("touch_reference_respects_cwd", |dirs, _sandbox| { + nu!( + cwd: dirs.test(), + "mkdir 'dir'; cd 'dir'; touch 'ref.txt'; touch --reference 'ref.txt' 'foo.txt'" + ); + + let path = dirs.test().join("dir/foo.txt"); + assert!(path.exists()); + }) +} diff --git a/crates/nu-command/tests/commands/use_.rs b/crates/nu-command/tests/commands/use_.rs index 8cb783f96b..63b1f8391a 100644 --- a/crates/nu-command/tests/commands/use_.rs +++ b/crates/nu-command/tests/commands/use_.rs @@ -189,9 +189,7 @@ fn use_module_creates_accurate_did_you_mean_1() { let actual = nu!(r#" module spam { export def foo [] { "foo" } }; use spam; foo "#); - assert!(actual.err.contains( - "command 'foo' was not found but it was imported from module 'spam'; try using `spam foo`" - )); + assert!(actual.err.contains("Did you mean `spam foo`")); } #[test] @@ -199,9 +197,9 @@ fn use_module_creates_accurate_did_you_mean_2() { let actual = nu!(r#" module spam { export def foo [] { "foo" } }; foo "#); - assert!(actual.err.contains( - "command 'foo' was not found but it exists in module 'spam'; try importing it with `use`" - )); + assert!(actual + .err + .contains("A command with that name exists in module `spam`")); } #[test] diff --git a/crates/nu-command/tests/format_conversions/json.rs b/crates/nu-command/tests/format_conversions/json.rs index 1989000df6..643e4c3f22 100644 --- a/crates/nu-command/tests/format_conversions/json.rs +++ b/crates/nu-command/tests/format_conversions/json.rs @@ -96,6 +96,32 @@ fn from_json_text_recognizing_objects_independently_to_table() { }) } +#[test] +fn from_json_text_objects_is_stream() { + Playground::setup("filter_from_json_test_2_is_stream", |dirs, sandbox| { + sandbox.with_files(&[FileWithContentToBeTrimmed( + "katz.txt", + r#" + {"name": "Yehuda", "rusty_luck": 1} + {"name": "JT", "rusty_luck": 1} + {"name": "Andres", "rusty_luck": 1} + {"name":"GorbyPuff", "rusty_luck": 3} + "#, + )]); + + let actual = nu!( + cwd: dirs.test(), pipeline( + r#" + open katz.txt + | from json -o + | describe -n + "# + )); + + assert_eq!(actual.out, "stream"); + }) +} + #[test] fn from_json_text_recognizing_objects_independently_to_table_strict() { Playground::setup("filter_from_json_test_2_strict", |dirs, sandbox| { diff --git a/crates/nu-command/tests/main.rs b/crates/nu-command/tests/main.rs index 72ccee3d85..1039e1a0d7 100644 --- a/crates/nu-command/tests/main.rs +++ b/crates/nu-command/tests/main.rs @@ -151,9 +151,8 @@ fn commands_declare_input_output_types() { let sig_name = cmd.signature().name; let category = cmd.signature().category; - if matches!(category, Category::Removed | Category::Custom(_)) { + if matches!(category, Category::Removed) { // Deprecated/Removed commands don't have to conform - // TODO: also upgrade the `--features dataframe` commands continue; } diff --git a/crates/nu-engine/Cargo.toml b/crates/nu-engine/Cargo.toml index 39fefc525e..7bf5ffe9d5 100644 --- a/crates/nu-engine/Cargo.toml +++ b/crates/nu-engine/Cargo.toml @@ -5,16 +5,16 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-engine" edition = "2021" license = "MIT" name = "nu-engine" -version = "0.93.1" +version = "0.94.1" [lib] bench = false [dependencies] -nu-protocol = { path = "../nu-protocol", features = ["plugin"], version = "0.93.1" } -nu-path = { path = "../nu-path", version = "0.93.1" } -nu-glob = { path = "../nu-glob", version = "0.93.1" } -nu-utils = { path = "../nu-utils", version = "0.93.1" } +nu-protocol = { path = "../nu-protocol", features = ["plugin"], version = "0.94.1" } +nu-path = { path = "../nu-path", version = "0.94.1" } +nu-glob = { path = "../nu-glob", version = "0.94.1" } +nu-utils = { path = "../nu-utils", version = "0.94.1" } [features] -plugin = [] +plugin = [] \ No newline at end of file diff --git a/crates/nu-engine/src/command_prelude.rs b/crates/nu-engine/src/command_prelude.rs index 089a2fb8fa..112f280db5 100644 --- a/crates/nu-engine/src/command_prelude.rs +++ b/crates/nu-engine/src/command_prelude.rs @@ -2,7 +2,7 @@ pub use crate::CallExt; pub use nu_protocol::{ ast::{Call, CellPath}, engine::{Command, EngineState, Stack}, - record, Category, ErrSpan, Example, IntoInterruptiblePipelineData, IntoPipelineData, - IntoSpanned, PipelineData, Record, ShellError, Signature, Span, Spanned, SyntaxShape, Type, - Value, + record, ByteStream, ByteStreamType, Category, ErrSpan, Example, IntoInterruptiblePipelineData, + IntoPipelineData, IntoSpanned, PipelineData, Record, ShellError, Signature, Span, Spanned, + SyntaxShape, Type, Value, }; diff --git a/crates/nu-engine/src/documentation.rs b/crates/nu-engine/src/documentation.rs index 62e68eaa6c..3cd1130060 100644 --- a/crates/nu-engine/src/documentation.rs +++ b/crates/nu-engine/src/documentation.rs @@ -2,18 +2,16 @@ use crate::eval_call; use nu_protocol::{ ast::{Argument, Call, Expr, Expression, RecordItem}, debugger::WithoutDebug, - engine::{EngineState, Stack}, + engine::{Command, EngineState, Stack}, record, Category, Example, IntoPipelineData, PipelineData, Signature, Span, SyntaxShape, Type, Value, }; use std::{collections::HashMap, fmt::Write}; pub fn get_full_help( - sig: &Signature, - examples: &[Example], + command: &dyn Command, engine_state: &EngineState, stack: &mut Stack, - is_parser_keyword: bool, ) -> String { let config = engine_state.get_config(); let doc_config = DocumentationConfig { @@ -23,14 +21,15 @@ pub fn get_full_help( }; let stack = &mut stack.start_capture(); + let signature = command.signature().update_from_command(command); get_documentation( - sig, - examples, + &signature, + &command.examples(), engine_state, stack, &doc_config, - is_parser_keyword, + command.is_keyword(), ) } @@ -61,7 +60,6 @@ fn nu_highlight_string(code_string: &str, engine_state: &EngineState, stack: &mu code_string.to_string() } -#[allow(clippy::cognitive_complexity)] fn get_documentation( sig: &Signature, examples: &[Example], diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index 0bc0c3727c..af051a1dc7 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -27,19 +27,9 @@ pub fn eval_call( let decl = engine_state.get_decl(call.decl_id); if !decl.is_known_external() && call.named_iter().any(|(flag, _, _)| flag.item == "help") { - let mut signature = engine_state.get_signature(decl); - signature.usage = decl.usage().to_string(); - signature.extra_usage = decl.extra_usage().to_string(); - - let full_help = get_full_help( - &signature, - &decl.examples(), - engine_state, - caller_stack, - decl.is_parser_keyword(), - ); - Ok(Value::string(full_help, call.head).into_pipeline_data()) - } else if let Some(block_id) = decl.get_block_id() { + let help = get_full_help(decl, engine_state, caller_stack); + Ok(Value::string(help, call.head).into_pipeline_data()) + } else if let Some(block_id) = decl.block_id() { let block = engine_state.get_block(block_id); let mut callee_stack = caller_stack.gather_captures(engine_state, &block.captures); diff --git a/crates/nu-engine/src/scope.rs b/crates/nu-engine/src/scope.rs index 1f5bf2a358..b6a43ca47c 100644 --- a/crates/nu-engine/src/scope.rs +++ b/crates/nu-engine/src/scope.rs @@ -111,13 +111,8 @@ impl<'e, 's> ScopeData<'e, 's> { "signatures" => self.collect_signatures(&signature, span), "usage" => Value::string(decl.usage(), span), "examples" => Value::list(examples, span), - // we can only be a is_builtin or is_custom, not both - "is_builtin" => Value::bool(!decl.is_custom_command(), span), + "type" => Value::string(decl.command_type().to_string(), span), "is_sub" => Value::bool(decl.is_sub(), span), - "is_plugin" => Value::bool(decl.is_plugin(), span), - "is_custom" => Value::bool(decl.is_custom_command(), span), - "is_keyword" => Value::bool(decl.is_parser_keyword(), span), - "is_extern" => Value::bool(decl.is_known_external(), span), "creates_scope" => Value::bool(signature.creates_scope, span), "extra_usage" => Value::string(decl.extra_usage(), span), "search_terms" => Value::string(decl.search_terms().join(", "), span), diff --git a/crates/nu-explore/Cargo.toml b/crates/nu-explore/Cargo.toml index 46756e4101..006e301859 100644 --- a/crates/nu-explore/Cargo.toml +++ b/crates/nu-explore/Cargo.toml @@ -5,21 +5,21 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-explore" edition = "2021" license = "MIT" name = "nu-explore" -version = "0.93.1" +version = "0.94.1" [lib] bench = false [dependencies] -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-color-config = { path = "../nu-color-config", version = "0.93.1" } -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-table = { path = "../nu-table", version = "0.93.1" } -nu-json = { path = "../nu-json", version = "0.93.1" } -nu-utils = { path = "../nu-utils", version = "0.93.1" } +nu-protocol = { path = "../nu-protocol", version = "0.94.1" } +nu-parser = { path = "../nu-parser", version = "0.94.1" } +nu-color-config = { path = "../nu-color-config", version = "0.94.1" } +nu-engine = { path = "../nu-engine", version = "0.94.1" } +nu-table = { path = "../nu-table", version = "0.94.1" } +nu-json = { path = "../nu-json", version = "0.94.1" } +nu-utils = { path = "../nu-utils", version = "0.94.1" } nu-ansi-term = { workspace = true } -nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.93.1" } +nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.94.1" } anyhow = { workspace = true } log = { workspace = true } @@ -32,4 +32,4 @@ ansi-str = { workspace = true } unicode-width = { workspace = true } lscolors = { workspace = true, default-features = false, features = [ "nu-ansi-term", -] } +] } \ No newline at end of file diff --git a/crates/nu-explore/src/commands/table.rs b/crates/nu-explore/src/commands/table.rs index 7b778a8971..39ef8f0f99 100644 --- a/crates/nu-explore/src/commands/table.rs +++ b/crates/nu-explore/src/commands/table.rs @@ -25,8 +25,6 @@ struct TableSettings { selected_column_s: Option