diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2225848cb4..2193598686 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -18,6 +18,14 @@ updates: ignore: - dependency-name: "*" update-types: ["version-update:semver-patch"] + groups: + # Only update polars as a whole as there are many subcrates that need to + # be updated at once. We explicitly depend on some of them, so batch their + # updates to not take up dependabot PR slots with dysfunctional PRs + polars: + patterns: + - "polars" + - "polars-*" - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index e8a7f55817..694ff6e1bb 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -19,7 +19,7 @@ jobs: # Prevent sudden announcement of a new advisory from failing ci: continue-on-error: true steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.7 - uses: rustsec/audit-check@v1.4.1 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4815491854..25fcfde55e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,76 +29,50 @@ jobs: # instead of 14 GB) which is too little for us right now. Revisit when `dfr` commands are # removed and we're only building the `polars` plugin instead platform: [windows-latest, macos-13, ubuntu-20.04] - feature: [default, dataframe] - include: - - feature: default - flags: "" - - feature: dataframe - flags: "--features=dataframe" - exclude: - - platform: windows-latest - feature: dataframe - - platform: macos-13 - feature: dataframe runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.7 - name: Setup Rust toolchain and cache - uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - with: - rustflags: "" + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 - name: cargo fmt run: cargo fmt --all -- --check # If changing these settings also change toolkit.nu - name: Clippy - run: cargo clippy --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- $CLIPPY_OPTIONS + run: cargo clippy --workspace --exclude nu_plugin_* -- $CLIPPY_OPTIONS # In tests we don't have to deny unwrap - name: Clippy of tests - run: cargo clippy --tests --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings + run: cargo clippy --tests --workspace --exclude nu_plugin_* -- -D warnings - name: Clippy of benchmarks - run: cargo clippy --benches --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings + run: cargo clippy --benches --workspace --exclude nu_plugin_* -- -D warnings tests: strategy: fail-fast: true matrix: platform: [windows-latest, macos-latest, ubuntu-20.04] - feature: [default, dataframe] include: - # linux CI cannot handle clipboard feature - default-flags: "" - - platform: ubuntu-20.04 + # linux CI cannot handle clipboard feature + - platform: ubuntu-20.04 default-flags: "--no-default-features --features=default-no-clipboard" - - feature: default - flags: "" - - feature: dataframe - flags: "--features=dataframe" - exclude: - - platform: windows-latest - feature: dataframe - - platform: macos-latest - feature: dataframe runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.7 - name: Setup Rust toolchain and cache - uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - with: - rustflags: "" + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 - name: Tests - run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }} ${{ matrix.flags }} - + run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }} - name: Check for clean repo shell: bash run: | @@ -121,12 +95,10 @@ jobs: runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.7 - name: Setup Rust toolchain and cache - uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - with: - rustflags: "" + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 - name: Install Nushell run: cargo install --path . --locked --no-default-features @@ -174,12 +146,10 @@ jobs: runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.7 - name: Setup Rust toolchain and cache - uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - with: - rustflags: "" + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 - name: Clippy run: cargo clippy --package nu_plugin_* -- $CLIPPY_OPTIONS diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index ab9f93d97d..9b7f44feba 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -27,7 +27,7 @@ jobs: # if: github.repository == 'nushell/nightly' steps: - name: Checkout - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.7 if: github.repository == 'nushell/nightly' with: ref: main @@ -36,10 +36,10 @@ jobs: token: ${{ secrets.WORKFLOW_TOKEN }} - name: Setup Nushell - uses: hustcer/setup-nu@v3.10 + uses: hustcer/setup-nu@v3.12 if: github.repository == 'nushell/nightly' with: - version: 0.93.0 + version: 0.95.0 # Synchronize the main branch of nightly repo with the main branch of Nushell official repo - name: Prepare for Nightly Release @@ -84,46 +84,35 @@ jobs: include: - target: aarch64-apple-darwin os: macos-latest - target_rustflags: '' - target: x86_64-apple-darwin os: macos-latest - target_rustflags: '' - target: x86_64-pc-windows-msvc extra: 'bin' os: windows-latest - target_rustflags: '' - target: x86_64-pc-windows-msvc extra: msi os: windows-latest - target_rustflags: '' - target: aarch64-pc-windows-msvc extra: 'bin' os: windows-latest - target_rustflags: '' - target: aarch64-pc-windows-msvc extra: msi os: windows-latest - target_rustflags: '' - target: x86_64-unknown-linux-gnu - os: ubuntu-20.04 - target_rustflags: '' + os: ubuntu-22.04 - target: x86_64-unknown-linux-musl - os: ubuntu-20.04 - target_rustflags: '' + os: ubuntu-22.04 - target: aarch64-unknown-linux-gnu - os: ubuntu-20.04 - target_rustflags: '' + os: ubuntu-22.04 - target: armv7-unknown-linux-gnueabihf - os: ubuntu-20.04 - target_rustflags: '' + os: ubuntu-22.04 - target: riscv64gc-unknown-linux-gnu os: ubuntu-latest - target_rustflags: '' runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.7 with: ref: main fetch-depth: 0 @@ -133,26 +122,24 @@ jobs: echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml - name: Setup Rust toolchain and cache - uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135` + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 + # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135` with: rustflags: '' - name: Setup Nushell - uses: hustcer/setup-nu@v3.10 + uses: hustcer/setup-nu@v3.12 with: - version: 0.93.0 + version: 0.95.0 - name: Release Nu Binary id: nu run: nu .github/workflows/release-pkg.nu env: - RELEASE_TYPE: standard OS: ${{ matrix.os }} REF: ${{ github.ref }} TARGET: ${{ matrix.target }} _EXTRA_: ${{ matrix.extra }} - TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }} - name: Create an Issue for Release Failure if: ${{ failure() }} @@ -174,7 +161,7 @@ jobs: # REF: https://github.com/marketplace/actions/gh-release # Create a release only in nushell/nightly repo - name: Publish Archive - uses: softprops/action-gh-release@v2.0.5 + uses: softprops/action-gh-release@v2.0.6 if: ${{ startsWith(github.repository, 'nushell/nightly') }} with: prerelease: true @@ -184,122 +171,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - full: - name: Full - needs: prepare - strategy: - fail-fast: false - matrix: - target: - - aarch64-apple-darwin - - x86_64-apple-darwin - - x86_64-pc-windows-msvc - - aarch64-pc-windows-msvc - - x86_64-unknown-linux-gnu - - x86_64-unknown-linux-musl - - aarch64-unknown-linux-gnu - extra: ['bin'] - include: - - target: aarch64-apple-darwin - os: macos-latest - target_rustflags: '--features=dataframe' - - target: x86_64-apple-darwin - os: macos-latest - target_rustflags: '--features=dataframe' - - target: x86_64-pc-windows-msvc - extra: 'bin' - os: windows-latest - target_rustflags: '--features=dataframe' - - target: x86_64-pc-windows-msvc - extra: msi - os: windows-latest - target_rustflags: '--features=dataframe' - - target: aarch64-pc-windows-msvc - extra: 'bin' - os: windows-latest - target_rustflags: '--features=dataframe' - - target: aarch64-pc-windows-msvc - extra: msi - os: windows-latest - target_rustflags: '--features=dataframe' - - target: x86_64-unknown-linux-gnu - os: ubuntu-20.04 - target_rustflags: '--features=dataframe' - - target: x86_64-unknown-linux-musl - os: ubuntu-20.04 - target_rustflags: '--features=dataframe' - - target: aarch64-unknown-linux-gnu - os: ubuntu-20.04 - target_rustflags: '--features=dataframe' - - runs-on: ${{matrix.os}} - - steps: - - uses: actions/checkout@v4.1.5 - with: - ref: main - fetch-depth: 0 - - - name: Update Rust Toolchain Target - run: | - echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml - - - name: Setup Rust toolchain and cache - uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135` - with: - rustflags: '' - - - name: Setup Nushell - uses: hustcer/setup-nu@v3.10 - with: - version: 0.93.0 - - - name: Release Nu Binary - id: nu - run: nu .github/workflows/release-pkg.nu - env: - RELEASE_TYPE: full - OS: ${{ matrix.os }} - REF: ${{ github.ref }} - TARGET: ${{ matrix.target }} - _EXTRA_: ${{ matrix.extra }} - TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }} - - - name: Create an Issue for Release Failure - if: ${{ failure() }} - uses: JasonEtco/create-an-issue@v2.9.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - update_existing: true - search_existing: open - filename: .github/AUTO_ISSUE_TEMPLATE/nightly-build-fail.md - - - name: Set Outputs of Short SHA - id: vars - run: | - echo "date=$(date -u +'%Y-%m-%d')" >> $GITHUB_OUTPUT - sha_short=$(git rev-parse --short HEAD) - echo "sha_short=${sha_short:0:7}" >> $GITHUB_OUTPUT - - # REF: https://github.com/marketplace/actions/gh-release - # Create a release only in nushell/nightly repo - - name: Publish Archive - uses: softprops/action-gh-release@v2.0.5 - if: ${{ startsWith(github.repository, 'nushell/nightly') }} - with: - draft: false - prerelease: true - name: Nu-nightly-${{ steps.vars.outputs.date }}-${{ steps.vars.outputs.sha_short }} - tag_name: nightly-${{ steps.vars.outputs.sha_short }} - body: | - This is a NIGHTLY build of Nushell. - It is NOT recommended for production use. - files: ${{ steps.nu.outputs.archive }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - cleanup: name: Cleanup # Should only run in nushell/nightly repo @@ -310,14 +181,14 @@ jobs: - name: Waiting for Release run: sleep 1800 - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.7 with: ref: main - name: Setup Nushell - uses: hustcer/setup-nu@v3.10 + uses: hustcer/setup-nu@v3.12 with: - version: 0.93.0 + version: 0.95.0 # Keep the last a few releases - name: Delete Older Releases diff --git a/.github/workflows/release-pkg.nu b/.github/workflows/release-pkg.nu index 046ea2475e..da2779f6d1 100755 --- a/.github/workflows/release-pkg.nu +++ b/.github/workflows/release-pkg.nu @@ -9,7 +9,6 @@ # Instructions for manually creating an MSI for Winget Releases when they fail # Added 2022-11-29 when Windows packaging wouldn't work # Updated again on 2023-02-23 because msis are still failing validation -# Update on 2023-10-18 to use RELEASE_TYPE env var to determine if full or not # To run this manual for windows here are the steps I take # checkout the release you want to publish # 1. git checkout 0.86.0 @@ -17,28 +16,26 @@ # 2. $env:CARGO_TARGET_DIR = "" # 2. hide-env CARGO_TARGET_DIR # 3. $env.TARGET = 'x86_64-pc-windows-msvc' -# 4. $env.TARGET_RUSTFLAGS = '' -# 5. $env.GITHUB_WORKSPACE = 'D:\nushell' -# 6. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt' -# 7. $env.OS = 'windows-latest' -# 8. $env.RELEASE_TYPE = '' # There is full and '' for normal releases +# 4. $env.GITHUB_WORKSPACE = 'D:\nushell' +# 5. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt' +# 6. $env.OS = 'windows-latest' # make sure 7z.exe is in your path https://www.7-zip.org/download.html -# 9. $env.Path = ($env.Path | append 'c:\apps\7-zip') +# 7. $env.Path = ($env.Path | append 'c:\apps\7-zip') # make sure aria2c.exe is in your path https://github.com/aria2/aria2 -# 10. $env.Path = ($env.Path | append 'c:\path\to\aria2c') +# 8. $env.Path = ($env.Path | append 'c:\path\to\aria2c') # make sure you have the wixtools installed https://wixtoolset.org/ -# 11. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools') +# 9. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools') # You need to run the release-pkg twice. The first pass, with _EXTRA_ as 'bin', makes the output # folder and builds everything. The second pass, that generates the msi file, with _EXTRA_ as 'msi' -# 12. $env._EXTRA_ = 'bin' -# 13. source .github\workflows\release-pkg.nu -# 14. cd .. -# 15. $env._EXTRA_ = 'msi' -# 16. source .github\workflows\release-pkg.nu +# 10. $env._EXTRA_ = 'bin' +# 11. source .github\workflows\release-pkg.nu +# 12. cd .. +# 13. $env._EXTRA_ = 'msi' +# 14. source .github\workflows\release-pkg.nu # After msi is generated, you have to update winget-pkgs repo, you'll need to patch the release # by deleting the existing msi and uploading this new msi. Then you'll need to update the hash # on the winget-pkgs PR. To generate the hash, run this command -# 17. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256 +# 15. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256 # Then, just take the output and put it in the winget-pkgs PR for the hash on the msi @@ -48,31 +45,15 @@ let os = $env.OS let target = $env.TARGET # Repo source dir like `/home/runner/work/nushell/nushell` let src = $env.GITHUB_WORKSPACE -let flags = $env.TARGET_RUSTFLAGS let dist = $'($env.GITHUB_WORKSPACE)/output' let version = (open Cargo.toml | get package.version) print $'Debugging info:' -print { version: $version, bin: $bin, os: $os, releaseType: $env.RELEASE_TYPE, target: $target, src: $src, flags: $flags, dist: $dist }; hr-line -b - -# Rename the full release name so that we won't break the existing scripts for standard release downloading, such as: -# curl -s https://api.github.com/repos/chmln/sd/releases/latest | grep browser_download_url | cut -d '"' -f 4 | grep x86_64-unknown-linux-musl -const FULL_RLS_NAMING = { - x86_64-apple-darwin: 'x86_64-darwin-full', - aarch64-apple-darwin: 'aarch64-darwin-full', - x86_64-unknown-linux-gnu: 'x86_64-linux-gnu-full', - x86_64-pc-windows-msvc: 'x86_64-windows-msvc-full', - x86_64-unknown-linux-musl: 'x86_64-linux-musl-full', - aarch64-unknown-linux-gnu: 'aarch64-linux-gnu-full', - aarch64-pc-windows-msvc: 'aarch64-windows-msvc-full', - riscv64gc-unknown-linux-gnu: 'riscv64-linux-gnu-full', - armv7-unknown-linux-gnueabihf: 'armv7-linux-gnueabihf-full', -} +print { version: $version, bin: $bin, os: $os, target: $target, src: $src, dist: $dist }; hr-line -b # $env let USE_UBUNTU = $os starts-with ubuntu -let FULL_NAME = $FULL_RLS_NAMING | get -i $target | default 'unknown-target-full' print $'(char nl)Packaging ($bin) v($version) for ($target) in ($src)...'; hr-line -b if not ('Cargo.lock' | path exists) { cargo generate-lockfile } @@ -91,23 +72,23 @@ if $os in ['macos-latest'] or $USE_UBUNTU { 'aarch64-unknown-linux-gnu' => { sudo apt-get install gcc-aarch64-linux-gnu -y $env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = 'aarch64-linux-gnu-gcc' - cargo-build-nu $flags + cargo-build-nu } 'riscv64gc-unknown-linux-gnu' => { sudo apt-get install gcc-riscv64-linux-gnu -y $env.CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER = 'riscv64-linux-gnu-gcc' - cargo-build-nu $flags + cargo-build-nu } 'armv7-unknown-linux-gnueabihf' => { sudo apt-get install pkg-config gcc-arm-linux-gnueabihf -y $env.CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER = 'arm-linux-gnueabihf-gcc' - cargo-build-nu $flags + cargo-build-nu } _ => { # musl-tools to fix 'Failed to find tool. Is `musl-gcc` installed?' # Actually just for x86_64-unknown-linux-musl target if $USE_UBUNTU { sudo apt install musl-tools -y } - cargo-build-nu $flags + cargo-build-nu } } } @@ -116,7 +97,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU { # Build for Windows without static-link-openssl feature # ---------------------------------------------------------------------------- if $os in ['windows-latest'] { - cargo-build-nu $flags + cargo-build-nu } # ---------------------------------------------------------------------------- @@ -162,7 +143,7 @@ cd $dist; print $'(char nl)Creating release archive...'; hr-line if $os in ['macos-latest'] or $USE_UBUNTU { let files = (ls | get name) - let dest = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' } + let dest = $'($bin)-($version)-($target)' let archive = $'($dist)/($dest).tar.gz' mkdir $dest @@ -177,7 +158,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU { } else if $os == 'windows-latest' { - let releaseStem = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' } + let releaseStem = $'($bin)-($version)-($target)' print $'(char nl)Download less related stuffs...'; hr-line aria2c https://github.com/jftuga/less-Windows/releases/download/less-v608/less.exe -o less.exe @@ -214,19 +195,11 @@ if $os in ['macos-latest'] or $USE_UBUNTU { } } -def 'cargo-build-nu' [ options: string ] { - if ($options | str trim | is-empty) { - if $os == 'windows-latest' { - cargo build --release --all --target $target - } else { - cargo build --release --all --target $target --features=static-link-openssl - } +def 'cargo-build-nu' [] { + if $os == 'windows-latest' { + cargo build --release --all --target $target } else { - if $os == 'windows-latest' { - cargo build --release --all --target $target $options - } else { - cargo build --release --all --target $target --features=static-link-openssl $options - } + cargo build --release --all --target $target --features=static-link-openssl } } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ffe653bd22..34cad24fbf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,167 +34,64 @@ jobs: include: - target: aarch64-apple-darwin os: macos-latest - target_rustflags: '' - target: x86_64-apple-darwin os: macos-latest - target_rustflags: '' - target: x86_64-pc-windows-msvc extra: 'bin' os: windows-latest - target_rustflags: '' - target: x86_64-pc-windows-msvc extra: msi os: windows-latest - target_rustflags: '' - target: aarch64-pc-windows-msvc extra: 'bin' os: windows-latest - target_rustflags: '' - target: aarch64-pc-windows-msvc extra: msi os: windows-latest - target_rustflags: '' - target: x86_64-unknown-linux-gnu - os: ubuntu-20.04 - target_rustflags: '' + os: ubuntu-22.04 - target: x86_64-unknown-linux-musl - os: ubuntu-20.04 - target_rustflags: '' + os: ubuntu-22.04 - target: aarch64-unknown-linux-gnu - os: ubuntu-20.04 - target_rustflags: '' + os: ubuntu-22.04 - target: armv7-unknown-linux-gnueabihf - os: ubuntu-20.04 - target_rustflags: '' + os: ubuntu-22.04 - target: riscv64gc-unknown-linux-gnu os: ubuntu-latest - target_rustflags: '' runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v4.1.5 + - uses: actions/checkout@v4.1.7 - name: Update Rust Toolchain Target run: | echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml - name: Setup Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135` + uses: actions-rust-lang/setup-rust-toolchain@v1.9.0 + # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135` with: cache: false rustflags: '' - name: Setup Nushell - uses: hustcer/setup-nu@v3.10 + uses: hustcer/setup-nu@v3.12 with: - version: 0.93.0 + version: 0.95.0 - name: Release Nu Binary id: nu run: nu .github/workflows/release-pkg.nu env: - RELEASE_TYPE: standard OS: ${{ matrix.os }} REF: ${{ github.ref }} TARGET: ${{ matrix.target }} _EXTRA_: ${{ matrix.extra }} - TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }} # REF: https://github.com/marketplace/actions/gh-release - name: Publish Archive - uses: softprops/action-gh-release@v2.0.5 - if: ${{ startsWith(github.ref, 'refs/tags/') }} - with: - draft: true - files: ${{ steps.nu.outputs.archive }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - full: - name: Full - - strategy: - fail-fast: false - matrix: - target: - - aarch64-apple-darwin - - x86_64-apple-darwin - - x86_64-pc-windows-msvc - - aarch64-pc-windows-msvc - - x86_64-unknown-linux-gnu - - x86_64-unknown-linux-musl - - aarch64-unknown-linux-gnu - extra: ['bin'] - include: - - target: aarch64-apple-darwin - os: macos-latest - target_rustflags: '--features=dataframe' - - target: x86_64-apple-darwin - os: macos-latest - target_rustflags: '--features=dataframe' - - target: x86_64-pc-windows-msvc - extra: 'bin' - os: windows-latest - target_rustflags: '--features=dataframe' - - target: x86_64-pc-windows-msvc - extra: msi - os: windows-latest - target_rustflags: '--features=dataframe' - - target: aarch64-pc-windows-msvc - extra: 'bin' - os: windows-latest - target_rustflags: '--features=dataframe' - - target: aarch64-pc-windows-msvc - extra: msi - os: windows-latest - target_rustflags: '--features=dataframe' - - target: x86_64-unknown-linux-gnu - os: ubuntu-20.04 - target_rustflags: '--features=dataframe' - - target: x86_64-unknown-linux-musl - os: ubuntu-20.04 - target_rustflags: '--features=dataframe' - - target: aarch64-unknown-linux-gnu - os: ubuntu-20.04 - target_rustflags: '--features=dataframe' - - runs-on: ${{matrix.os}} - - steps: - - uses: actions/checkout@v4.1.5 - - - name: Update Rust Toolchain Target - run: | - echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml - - - name: Setup Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@v1.8.0 - # WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135` - with: - cache: false - rustflags: '' - - - name: Setup Nushell - uses: hustcer/setup-nu@v3.10 - with: - version: 0.93.0 - - - name: Release Nu Binary - id: nu - run: nu .github/workflows/release-pkg.nu - env: - RELEASE_TYPE: full - OS: ${{ matrix.os }} - REF: ${{ github.ref }} - TARGET: ${{ matrix.target }} - _EXTRA_: ${{ matrix.extra }} - TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }} - - # REF: https://github.com/marketplace/actions/gh-release - - name: Publish Archive - uses: softprops/action-gh-release@v2.0.5 + uses: softprops/action-gh-release@v2.0.6 if: ${{ startsWith(github.ref, 'refs/tags/') }} with: draft: true diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml index dacfb83928..0f1e4cb5d9 100644 --- a/.github/workflows/typos.yml +++ b/.github/workflows/typos.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Actions Repository - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.7 - name: Check spelling - uses: crate-ci/typos@v1.21.0 + uses: crate-ci/typos@v1.23.1 diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000..25731e87b2 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,26 @@ +cff-version: 1.2.0 +title: 'Nushell' +message: >- + If you use this software and wish to cite it, + you can use the metadata from this file. +type: software +authors: + - name: "The Nushell Project Team" +identifiers: + - type: url + value: 'https://github.com/nushell/nushell' + description: Repository +repository-code: 'https://github.com/nushell/nushell' +url: 'https://www.nushell.sh/' +abstract: >- + The goal of the Nushell project is to take the Unix + philosophy of shells, where pipes connect simple commands + together, and bring it to the modern style of development. + Thus, rather than being either a shell, or a programming + language, Nushell connects both by bringing a rich + programming language and a full-featured shell together + into one package. +keywords: + - nushell + - shell +license: MIT diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d5cf758b56..6d7c256d4c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,11 +71,6 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref cargo run ``` -- Build and run with dataframe support. - ```nushell - cargo run --features=dataframe - ``` - - Run Clippy on Nushell: ```nushell @@ -93,11 +88,6 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref cargo test --workspace ``` - along with dataframe tests - - ```nushell - cargo test --workspace --features=dataframe - ``` or via the `toolkit.nu` command: ```nushell use toolkit.nu test @@ -240,7 +230,7 @@ You can help us to make the review process a smooth experience: - Choose what simplifies having confidence in the conflict resolution and the review. **Merge commits in your branch are OK** in the squash model. - Feel free to notify your reviewers or affected PR authors if your change might cause larger conflicts with another change. - During the rollup of multiple PRs, we may choose to resolve merge conflicts and CI failures ourselves. (Allow maintainers to push to your branch to enable us to do this quickly.) - + ## License We use the [MIT License](https://github.com/nushell/nushell/blob/main/LICENSE) in all of our Nushell projects. If you are including or referencing a crate that uses the [GPL License](https://www.gnu.org/licenses/gpl-3.0.en.html#license-text) unfortunately we will not be able to accept your PR. diff --git a/Cargo.lock b/Cargo.lock index ad5a1e3a76..7243a801fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -377,7 +377,7 @@ dependencies = [ "bitflags 2.5.0", "cexpr", "clang-sys", - "itertools 0.11.0", + "itertools 0.12.1", "lazy_static", "lazycell", "proc-macro2", @@ -478,17 +478,6 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ada7f35ca622a86a4d6c27be2633fc6c243ecc834859628fcce0681d8e76e1c8" -[[package]] -name = "brotli" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor 2.5.1", -] - [[package]] name = "brotli" version = "5.0.0" @@ -497,17 +486,7 @@ checksum = "19483b140a7ac7174d34b5a581b406c64f84da5409d3e09cf4fff604f9270e67" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor 4.0.0", -] - -[[package]] -name = "brotli-decompressor" -version = "2.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", + "brotli-decompressor", ] [[package]] @@ -871,7 +850,7 @@ checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7" dependencies = [ "crossterm", "strum", - "strum_macros 0.26.2", + "strum_macros", "unicode-width", ] @@ -941,6 +920,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1129,6 +1117,36 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "curl" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e2161dd6eba090ff1594084e95fd67aeccf04382ffea77999ea94ed42ec67b6" +dependencies = [ + "curl-sys", + "libc", + "openssl-probe", + "openssl-sys", + "schannel", + "socket2", + "windows-sys 0.52.0", +] + +[[package]] +name = "curl-sys" +version = "0.4.73+curl-8.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "450ab250ecf17227c39afb9a2dd9261dc0035cb80f2612472fc0c4aac2dcb84d" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "windows-sys 0.52.0", +] + [[package]] name = "deranged" version = "0.3.11" @@ -1160,6 +1178,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "deunicode" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339544cc9e2c4dc3fc7149fd630c5f22263a4fdf18a98afd0075784968b5cf00" + [[package]] name = "dialoguer" version = "0.11.0" @@ -1239,6 +1263,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "doctest-file" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aac81fa3e28d21450aa4d2ac065992ba96a1d7303efbce51a95f4fd175b67562" + [[package]] name = "downcast-rs" version = "1.2.1" @@ -1295,6 +1325,9 @@ name = "either" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" +dependencies = [ + "serde", +] [[package]] name = "eml-parser" @@ -1333,6 +1366,16 @@ dependencies = [ "syn 2.0.60", ] +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + [[package]] name = "env_logger" version = "0.8.4" @@ -1343,6 +1386,19 @@ dependencies = [ "regex", ] +[[package]] +name = "env_logger" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -1696,9 +1752,9 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "git2" -version = "0.18.3" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70" +checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" dependencies = [ "bitflags 2.5.0", "libc", @@ -1794,6 +1850,7 @@ dependencies = [ "ahash 0.8.11", "allocator-api2", "rayon", + "serde", ] [[package]] @@ -1856,12 +1913,26 @@ checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" dependencies = [ "log", "mac", - "markup5ever", + "markup5ever 0.11.0", "proc-macro2", "quote", "syn 1.0.109", ] +[[package]] +name = "html5ever" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13771afe0e6e846f1e67d038d4cb29998a6779f93c809212e4e9c32efd244d4" +dependencies = [ + "log", + "mac", + "markup5ever 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.60", +] + [[package]] name = "http" version = "0.2.12" @@ -1908,6 +1979,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.28" @@ -1997,12 +2074,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "indoc" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" - [[package]] name = "inotify" version = "0.9.6" @@ -2034,10 +2105,11 @@ dependencies = [ [[package]] name = "interprocess" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4d0250d41da118226e55b3d50ca3f0d9e0a0f6829b92f543ac0054aeea1572" +checksum = "67bafc2f5dbdad79a6d925649758d5472647b416028099f0b829d1b67fdd47d3" dependencies = [ + "doctest-file", "libc", "recvmsg", "widestring", @@ -2296,9 +2368,9 @@ dependencies = [ [[package]] name = "libgit2-sys" -version = "0.16.2+1.7.2" +version = "0.17.0+1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" +checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224" dependencies = [ "cc", "libc", @@ -2310,9 +2382,9 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" dependencies = [ "cfg-if", "windows-targets 0.52.5", @@ -2326,9 +2398,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libmimalloc-sys" -version = "0.1.37" +version = "0.1.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81eb4061c0582dedea1cbc7aff2240300dd6982e0239d1c99e65c1dbf4a30ba7" +checksum = "0e7bb23d733dfcc8af652a78b7bf232f0e967710d044732185e561e47c0336b6" dependencies = [ "cc", "libc", @@ -2524,6 +2596,32 @@ dependencies = [ "tendril", ] +[[package]] +name = "markup5ever" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16ce3abbeba692c8b8441d036ef91aea6df8da2c6b6e21c7e14d3c18e526be45" +dependencies = [ + "log", + "phf 0.11.2", + "phf_codegen 0.11.2", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "markup5ever_rcdom" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edaa21ab3701bfee5099ade5f7e1f84553fd19228cf332f13cd6e964bf59be18" +dependencies = [ + "html5ever 0.27.0", + "markup5ever 0.12.1", + "tendril", + "xml5ever", +] + [[package]] name = "md-5" version = "0.10.6" @@ -2582,9 +2680,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.41" +version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f41a2280ded0da56c8cf898babb86e8f10651a34adcfff190ae9a1159c6908d" +checksum = "e9186d86b79b52f4a77af65604b51225e8db1d6ee7e3f41aec1e40829c71a176" dependencies = [ "libmimalloc-sys", ] @@ -2770,7 +2868,7 @@ dependencies = [ [[package]] name = "nu" -version = "0.93.1" +version = "0.95.1" dependencies = [ "assert_cmd", "crossterm", @@ -2782,7 +2880,6 @@ dependencies = [ "nix", "nu-cli", "nu-cmd-base", - "nu-cmd-dataframe", "nu-cmd-extra", "nu-cmd-lang", "nu-cmd-plugin", @@ -2803,6 +2900,7 @@ dependencies = [ "openssl", "pretty_assertions", "reedline", + "regex", "rstest", "serde_json", "serial_test", @@ -2824,7 +2922,7 @@ dependencies = [ [[package]] name = "nu-cli" -version = "0.93.1" +version = "0.95.1" dependencies = [ "chrono", "crossterm", @@ -2847,7 +2945,6 @@ dependencies = [ "nu-test-support", "nu-utils", "once_cell", - "pathdiff", "percent-encoding", "reedline", "rstest", @@ -2860,7 +2957,7 @@ dependencies = [ [[package]] name = "nu-cmd-base" -version = "0.93.1" +version = "0.95.1" dependencies = [ "indexmap", "miette", @@ -2870,32 +2967,9 @@ dependencies = [ "nu-protocol", ] -[[package]] -name = "nu-cmd-dataframe" -version = "0.93.1" -dependencies = [ - "chrono", - "chrono-tz 0.8.6", - "fancy-regex", - "indexmap", - "nu-cmd-lang", - "nu-engine", - "nu-parser", - "nu-protocol", - "num", - "polars", - "polars-arrow", - "polars-io", - "polars-ops", - "polars-plan", - "polars-utils", - "serde", - "sqlparser 0.45.0", -] - [[package]] name = "nu-cmd-extra" -version = "0.93.1" +version = "0.95.1" dependencies = [ "fancy-regex", "heck 0.5.0", @@ -2920,7 +2994,7 @@ dependencies = [ [[package]] name = "nu-cmd-lang" -version = "0.93.1" +version = "0.95.1" dependencies = [ "itertools 0.12.1", "nu-engine", @@ -2932,7 +3006,7 @@ dependencies = [ [[package]] name = "nu-cmd-plugin" -version = "0.93.1" +version = "0.95.1" dependencies = [ "itertools 0.12.1", "nu-engine", @@ -2943,7 +3017,7 @@ dependencies = [ [[package]] name = "nu-color-config" -version = "0.93.1" +version = "0.95.1" dependencies = [ "nu-ansi-term", "nu-engine", @@ -2955,12 +3029,12 @@ dependencies = [ [[package]] name = "nu-command" -version = "0.93.1" +version = "0.95.1" dependencies = [ "alphanumeric-sort", "base64 0.22.1", "bracoxide", - "brotli 5.0.0", + "brotli", "byteorder", "bytesize", "calamine", @@ -2970,6 +3044,7 @@ dependencies = [ "chrono-tz 0.8.6", "crossterm", "csv", + "deunicode", "dialoguer", "digest", "dirs-next", @@ -3037,6 +3112,7 @@ dependencies = [ "sha2", "sysinfo", "tabled", + "tempfile", "terminal_size", "titlecase", "toml 0.8.12", @@ -3052,7 +3128,7 @@ dependencies = [ "uu_mv", "uu_uname", "uu_whoami", - "uucore 0.0.25", + "uucore", "uuid", "v_htmlescape", "wax", @@ -3062,9 +3138,21 @@ dependencies = [ ] [[package]] -name = "nu-engine" -version = "0.93.1" +name = "nu-derive-value" +version = "0.95.1" dependencies = [ + "convert_case", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "nu-engine" +version = "0.95.1" +dependencies = [ + "log", "nu-glob", "nu-path", "nu-protocol", @@ -3073,7 +3161,7 @@ dependencies = [ [[package]] name = "nu-explore" -version = "0.93.1" +version = "0.95.1" dependencies = [ "ansi-str", "anyhow", @@ -3098,14 +3186,14 @@ dependencies = [ [[package]] name = "nu-glob" -version = "0.93.1" +version = "0.95.1" dependencies = [ "doc-comment", ] [[package]] name = "nu-json" -version = "0.93.1" +version = "0.95.1" dependencies = [ "linked-hash-map", "num-traits", @@ -3115,7 +3203,7 @@ dependencies = [ [[package]] name = "nu-lsp" -version = "0.93.1" +version = "0.95.1" dependencies = [ "assert-json-diff", "crossbeam-channel", @@ -3136,7 +3224,7 @@ dependencies = [ [[package]] name = "nu-parser" -version = "0.93.1" +version = "0.95.1" dependencies = [ "bytesize", "chrono", @@ -3152,7 +3240,7 @@ dependencies = [ [[package]] name = "nu-path" -version = "0.93.1" +version = "0.95.1" dependencies = [ "dirs-next", "omnipath", @@ -3161,7 +3249,7 @@ dependencies = [ [[package]] name = "nu-plugin" -version = "0.93.1" +version = "0.95.1" dependencies = [ "log", "nix", @@ -3176,7 +3264,7 @@ dependencies = [ [[package]] name = "nu-plugin-core" -version = "0.93.1" +version = "0.95.1" dependencies = [ "interprocess", "log", @@ -3190,7 +3278,7 @@ dependencies = [ [[package]] name = "nu-plugin-engine" -version = "0.93.1" +version = "0.95.1" dependencies = [ "log", "nu-engine", @@ -3205,7 +3293,7 @@ dependencies = [ [[package]] name = "nu-plugin-protocol" -version = "0.93.1" +version = "0.95.1" dependencies = [ "bincode", "nu-protocol", @@ -3217,7 +3305,7 @@ dependencies = [ [[package]] name = "nu-plugin-test-support" -version = "0.93.1" +version = "0.95.1" dependencies = [ "nu-ansi-term", "nu-cmd-lang", @@ -3235,7 +3323,7 @@ dependencies = [ [[package]] name = "nu-pretty-hex" -version = "0.93.1" +version = "0.95.1" dependencies = [ "heapless", "nu-ansi-term", @@ -3244,17 +3332,20 @@ dependencies = [ [[package]] name = "nu-protocol" -version = "0.93.1" +version = "0.95.1" dependencies = [ - "brotli 5.0.0", + "brotli", "byte-unit", "chrono", "chrono-humanize", + "convert_case", "fancy-regex", "indexmap", + "log", "lru", "miette", "nix", + "nu-derive-value", "nu-path", "nu-system", "nu-test-support", @@ -3267,7 +3358,7 @@ dependencies = [ "serde", "serde_json", "strum", - "strum_macros 0.26.2", + "strum_macros", "tempfile", "thiserror", "typetag", @@ -3275,7 +3366,7 @@ dependencies = [ [[package]] name = "nu-std" -version = "0.93.1" +version = "0.95.1" dependencies = [ "log", "miette", @@ -3286,9 +3377,10 @@ dependencies = [ [[package]] name = "nu-system" -version = "0.93.1" +version = "0.95.1" dependencies = [ "chrono", + "itertools 0.12.1", "libc", "libproc", "log", @@ -3303,7 +3395,7 @@ dependencies = [ [[package]] name = "nu-table" -version = "0.93.1" +version = "0.95.1" dependencies = [ "fancy-regex", "nu-ansi-term", @@ -3317,7 +3409,7 @@ dependencies = [ [[package]] name = "nu-term-grid" -version = "0.93.1" +version = "0.95.1" dependencies = [ "nu-utils", "unicode-width", @@ -3325,7 +3417,7 @@ dependencies = [ [[package]] name = "nu-test-support" -version = "0.93.1" +version = "0.95.1" dependencies = [ "nu-glob", "nu-path", @@ -3337,7 +3429,7 @@ dependencies = [ [[package]] name = "nu-utils" -version = "0.93.1" +version = "0.95.1" dependencies = [ "crossterm_winapi", "log", @@ -3363,7 +3455,7 @@ dependencies = [ [[package]] name = "nu_plugin_example" -version = "0.93.1" +version = "0.95.1" dependencies = [ "nu-cmd-lang", "nu-plugin", @@ -3373,7 +3465,7 @@ dependencies = [ [[package]] name = "nu_plugin_formats" -version = "0.93.1" +version = "0.95.1" dependencies = [ "eml-parser", "ical", @@ -3386,7 +3478,7 @@ dependencies = [ [[package]] name = "nu_plugin_gstat" -version = "0.93.1" +version = "0.95.1" dependencies = [ "git2", "nu-plugin", @@ -3395,7 +3487,7 @@ dependencies = [ [[package]] name = "nu_plugin_inc" -version = "0.93.1" +version = "0.95.1" dependencies = [ "nu-plugin", "nu-protocol", @@ -3404,12 +3496,15 @@ dependencies = [ [[package]] name = "nu_plugin_polars" -version = "0.93.1" +version = "0.95.1" dependencies = [ "chrono", "chrono-tz 0.9.0", + "env_logger 0.11.3", "fancy-regex", "indexmap", + "log", + "mimalloc", "nu-cmd-lang", "nu-command", "nu-engine", @@ -3418,16 +3513,16 @@ dependencies = [ "nu-plugin", "nu-plugin-test-support", "nu-protocol", + "nu-utils", "num", "polars", "polars-arrow", "polars-io", - "polars-lazy", "polars-ops", "polars-plan", "polars-utils", "serde", - "sqlparser 0.45.0", + "sqlparser", "tempfile", "typetag", "uuid", @@ -3435,19 +3530,22 @@ dependencies = [ [[package]] name = "nu_plugin_query" -version = "0.93.1" +version = "0.95.1" dependencies = [ "gjson", "nu-plugin", "nu-protocol", "scraper", + "serde", + "serde_json", "sxd-document", "sxd-xpath", + "webpage", ] [[package]] name = "nu_plugin_stress_internals" -version = "0.93.1" +version = "0.95.1" dependencies = [ "interprocess", "serde", @@ -3573,7 +3671,7 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "nuon" -version = "0.93.1" +version = "0.95.1" dependencies = [ "chrono", "fancy-regex", @@ -3670,9 +3768,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "open" -version = "5.1.2" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "449f0ff855d85ddbf1edd5b646d65249ead3f5e422aaa86b7d2d0b049b103e32" +checksum = "9d2c909a3fce3bd80efef4cd1c6c056bd9376a8fe06fcfdbebaf32cb485a7e37" dependencies = [ "is-wsl", "libc", @@ -3754,9 +3852,9 @@ dependencies = [ [[package]] name = "os_pipe" -version = "1.1.5" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57119c3b893986491ec9aa85056780d3a0f3cf4da7cc09dd3650dbd6c6738fb9" +checksum = "29d73ba8daf8fac13b0501d1abeddcfe21ba7401ada61a819144b6c2a4f32209" dependencies = [ "libc", "windows-sys 0.52.0", @@ -4037,9 +4135,9 @@ dependencies = [ [[package]] name = "polars" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea21b858b16b9c0e17a12db2800d11aa5b4bd182be6b3022eb537bbfc1f2db5" +checksum = "ce49e10a756f68eb99c102c6b2a0cbc0c583a0fa7263536ad0913d94be878d2d" dependencies = [ "getrandom", "polars-arrow", @@ -4057,9 +4155,9 @@ dependencies = [ [[package]] name = "polars-arrow" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "725b09f2b5ef31279b66e27bbab63c58d49d8f6696b66b1f46c7eaab95e80f75" +checksum = "b436f83f62e864f0d91871e26528f2c5552c7cf07c8d77547f1b8e3fde22bd27" dependencies = [ "ahash 0.8.11", "atoi", @@ -4105,9 +4203,9 @@ dependencies = [ [[package]] name = "polars-compute" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a796945b14b14fbb79b91ef0406e6fddca2be636e889f81ea5d6ee7d36efb4fe" +checksum = "f6758f834f07e622a2f859bebb542b2b7f8879b8704dbb2b2bbab460ddcdca4b" dependencies = [ "bytemuck", "either", @@ -4121,9 +4219,9 @@ dependencies = [ [[package]] name = "polars-core" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465f70d3e96b6d0b1a43c358ba451286b8c8bd56696feff020d65702aa33e35c" +checksum = "7ed262e9bdda15a12a9bfcfc9200bec5253335633dbd86cf5b94fda0194244b3" dependencies = [ "ahash 0.8.11", "bitflags 2.5.0", @@ -4155,9 +4253,9 @@ dependencies = [ [[package]] name = "polars-error" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5224d5d05e6b8a6f78b75951ae1b5f82c8ab1979e11ffaf5fd41941e3d5b0757" +checksum = "53e1707a17475ba5e74c349154b415e3148a1a275e395965427971b5e53ad621" dependencies = [ "avro-schema", "polars-arrow-format", @@ -4167,10 +4265,30 @@ dependencies = [ ] [[package]] -name = "polars-io" -version = "0.39.2" +name = "polars-expr" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2c8589e418cbe4a48228d64b2a8a40284a82ec3c98817c0c2bcc0267701338b" +checksum = "31a9688d5842e7a7fbad88e67a174778794a91d97d3bba1b3c09dd1656fee3b2" +dependencies = [ + "ahash 0.8.11", + "bitflags 2.5.0", + "once_cell", + "polars-arrow", + "polars-core", + "polars-io", + "polars-ops", + "polars-plan", + "polars-time", + "polars-utils", + "rayon", + "smartstring", +] + +[[package]] +name = "polars-io" +version = "0.41.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18798dacd94fb9263f65f63f0feab0908675422646d6f7fc37043b85ff6dca35" dependencies = [ "ahash 0.8.11", "async-trait", @@ -4209,9 +4327,9 @@ dependencies = [ [[package]] name = "polars-json" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81224492a649a12b668480c0cf219d703f432509765d2717e72fe32ad16fc701" +checksum = "044ea319f667efbf8007c4c38171c2956e0e7f9b078eb66e31e82f80d1e14b51" dependencies = [ "ahash 0.8.11", "chrono", @@ -4230,18 +4348,21 @@ dependencies = [ [[package]] name = "polars-lazy" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2632b1af668e2058d5f8f916d8fbde3cac63d03ae29a705f598e41dcfeb7f" +checksum = "74a11994c2211f2e99d9ac31776fd7c2c0607d5fe62d5b5db9e396f7d663f3d5" dependencies = [ "ahash 0.8.11", "bitflags 2.5.0", "glob", + "memchr", "once_cell", "polars-arrow", "polars-core", + "polars-expr", "polars-io", "polars-json", + "polars-mem-engine", "polars-ops", "polars-pipe", "polars-plan", @@ -4253,14 +4374,33 @@ dependencies = [ ] [[package]] -name = "polars-ops" -version = "0.39.2" +name = "polars-mem-engine" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efdbdb4d9a92109bc2e0ce8e17af5ae8ab643bb5b7ee9d1d74f0aeffd1fbc95f" +checksum = "5acd5fde6fadaddfcae3227ec5b64121007928f8e68870c80653438e20c1c587" +dependencies = [ + "polars-arrow", + "polars-core", + "polars-error", + "polars-expr", + "polars-io", + "polars-json", + "polars-ops", + "polars-plan", + "polars-time", + "polars-utils", + "rayon", +] + +[[package]] +name = "polars-ops" +version = "0.41.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4170c59e974727941edfb722f6d430ed623be9e7f30581ee00832c907f1b9fd" dependencies = [ "ahash 0.8.11", "argminmax", - "base64 0.21.7", + "base64 0.22.1", "bytemuck", "chrono", "chrono-tz 0.8.6", @@ -4290,14 +4430,14 @@ dependencies = [ [[package]] name = "polars-parquet" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b421d2196f786fdfe162db614c8485f8308fe41575d4de634a39bbe460d1eb6a" +checksum = "c684638c36c60c691d707d414249fe8af4a19a35a39d418464b140fe23732e5d" dependencies = [ "ahash 0.8.11", "async-stream", - "base64 0.21.7", - "brotli 3.5.0", + "base64 0.22.1", + "brotli", "ethnum", "flate2", "futures", @@ -4305,9 +4445,11 @@ dependencies = [ "num-traits", "parquet-format-safe", "polars-arrow", + "polars-compute", "polars-error", "polars-utils", "seq-macro", + "serde", "simdutf8", "snap", "streaming-decompression", @@ -4316,9 +4458,9 @@ dependencies = [ [[package]] name = "polars-pipe" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48700f1d5bd56a15451e581f465c09541492750360f18637b196f995470a015c" +checksum = "832af9fbebc4c074d95fb19e1ef9e1bf37c343641238c2476febff296a7028ea" dependencies = [ "crossbeam-channel", "crossbeam-queue", @@ -4328,6 +4470,7 @@ dependencies = [ "polars-arrow", "polars-compute", "polars-core", + "polars-expr", "polars-io", "polars-ops", "polars-plan", @@ -4341,13 +4484,14 @@ dependencies = [ [[package]] name = "polars-plan" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fb8e2302e20c44defd5be8cad9c96e75face63c3a5f609aced8c4ec3b3ac97d" +checksum = "801390ea815c05c9cf8337f3148090c9c10c9595a839fa0706b77cc2405b4466" dependencies = [ "ahash 0.8.11", "bytemuck", "chrono-tz 0.8.6", + "either", "hashbrown 0.14.5", "once_cell", "percent-encoding", @@ -4364,15 +4508,15 @@ dependencies = [ "regex", "serde", "smartstring", - "strum_macros 0.25.3", + "strum_macros", "version_check", ] [[package]] name = "polars-row" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a515bdc68c2ae3702e3de70d89601f3b71ca8137e282a226dddb53ee4bacfa2e" +checksum = "dee955e91b605fc91db4d0a8ea02609d3a09ff79256d905214a2a6f758cd6f7b" dependencies = [ "bytemuck", "polars-arrow", @@ -4382,29 +4526,33 @@ dependencies = [ [[package]] name = "polars-sql" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4bb7cc1c04c3023d1953b2f1dec50515e8fd8169a5a2bf4967b3b082232db7" +checksum = "d89c00a4b399501d5bd478e8e8022b9391047fe8570324ecba20c4e4833c0e87" dependencies = [ "hex", + "once_cell", "polars-arrow", "polars-core", "polars-error", "polars-lazy", + "polars-ops", "polars-plan", + "polars-time", "rand", "serde", "serde_json", - "sqlparser 0.39.0", + "sqlparser", ] [[package]] name = "polars-time" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efc18e3ad92eec55db89d88f16c22d436559ba7030cf76f86f6ed7a754b673f1" +checksum = "9689b3aff99d64befe300495528bdc44c36d2656c3a8b242a790d4f43df027fc" dependencies = [ "atoi", + "bytemuck", "chrono", "chrono-tz 0.8.6", "now", @@ -4421,9 +4569,9 @@ dependencies = [ [[package]] name = "polars-utils" -version = "0.39.2" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c760b6c698cfe2fbbbd93d6cfb408db14ececfe1d92445dae2229ce1b5b21ae8" +checksum = "12081e346983a91e26f395597e1d53dea1b4ecd694653aee1cc402d2fae01f04" dependencies = [ "ahash 0.8.11", "bytemuck", @@ -4659,7 +4807,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ - "env_logger", + "env_logger 0.8.4", "log", "rand", ] @@ -4738,21 +4886,21 @@ dependencies = [ [[package]] name = "ratatui" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a564a852040e82671dc50a37d88f3aa83bbc690dfc6844cfe7a2591620206a80" +checksum = "f44c9e68fd46eda15c646fbb85e1040b657a58cdc8c98db1d97a55930d991eef" dependencies = [ "bitflags 2.5.0", "cassowary", "compact_str", "crossterm", - "indoc", "itertools 0.12.1", "lru", "paste", "stability", "strum", "unicode-segmentation", + "unicode-truncate", "unicode-width", ] @@ -4843,7 +4991,7 @@ dependencies = [ [[package]] name = "reedline" version = "0.32.0" -source = "git+https://github.com/nushell/reedline?branch=main#a580ea56d4e5a889468b2969d2a1534379504ab6" +source = "git+https://github.com/nushell/reedline?branch=main#480059a3f52cf919341cda88e8c544edd846bc73" dependencies = [ "arboard", "chrono", @@ -4856,7 +5004,7 @@ dependencies = [ "serde_json", "strip-ansi-escapes", "strum", - "strum_macros 0.26.2", + "strum_macros", "thiserror", "unicode-segmentation", "unicode-width", @@ -5210,7 +5358,7 @@ dependencies = [ "ahash 0.8.11", "cssparser", "ego-tree", - "html5ever", + "html5ever 0.26.0", "once_cell", "selectors", "tendril", @@ -5426,9 +5574,9 @@ dependencies = [ [[package]] name = "shadow-rs" -version = "0.27.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7960cbd6ba74691bb15e7ebf97f7136bd02d1115f5695a58c1f31d5645750128" +checksum = "0a600f795d0894cda22235b44eea4b85c2a35b405f65523645ac8e35b306817a" dependencies = [ "const_format", "is_debug", @@ -5575,18 +5723,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.39.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743b4dc2cbde11890ccb254a8fc9d537fa41b36da00de2a1c5e9848c9bc42bd7" -dependencies = [ - "log", -] - -[[package]] -name = "sqlparser" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7bbffee862a796d67959a89859d6b1046bb5016d63e23835ad0da182777bbe0" +checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" dependencies = [ "log", ] @@ -5700,20 +5839,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" dependencies = [ - "strum_macros 0.26.2", -] - -[[package]] -name = "strum_macros" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.60", + "strum_macros", ] [[package]] @@ -6090,6 +6216,7 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" dependencies = [ + "indexmap", "serde", "serde_spanned", "toml_datetime", @@ -6311,6 +6438,16 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" +[[package]] +name = "unicode-truncate" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5fbabedabe362c618c714dbefda9927b5afc8e2a8102f47f081089a9019226" +dependencies = [ + "itertools 0.12.1", + "unicode-width", +] + [[package]] name = "unicode-width" version = "0.1.12" @@ -6378,98 +6515,82 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uu_cp" -version = "0.0.25" +version = "0.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcbe045dc92209114afdfd366bd18f7b95dbf999f3eaa85ad6dca910b0be3d56" +checksum = "6fb99d355ccb02e8c514e4a1d93e4aa4eedea9837de24635dfd24c165971444e" dependencies = [ "clap", "filetime", "indicatif", "libc", "quick-error 2.0.1", - "uucore 0.0.26", + "uucore", "walkdir", "xattr", ] [[package]] name = "uu_mkdir" -version = "0.0.25" +version = "0.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "040aa4584036b2f65e05387b0ea9ac468afce1db325743ce5f350689fd9ce4ae" +checksum = "219588fbc146f18188781208ac4034616c51cf151677b4e1f9caf63ca8a7f2cf" dependencies = [ "clap", - "uucore 0.0.26", + "uucore", ] [[package]] name = "uu_mktemp" -version = "0.0.25" +version = "0.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f240a99c36d768153874d198c43605a45c86996b576262689a0f18248cc3bc57" +checksum = "b1e79ad2c5911908fce23a6069c52ca82e1997e2ed4bf6abf2d867c79c3dc73f" dependencies = [ "clap", "rand", "tempfile", - "uucore 0.0.26", + "uucore", ] [[package]] name = "uu_mv" -version = "0.0.25" +version = "0.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c99fd7c75e6e85553c92537314be3d9a64b4927051aa1608513feea2f933022" +checksum = "cd57c8d02f8a99ed56ed9f6fddab403ee0e2bf9e8f3a5ca8f0f9e4d6e3e392a0" dependencies = [ "clap", "fs_extra", "indicatif", - "uucore 0.0.26", + "uucore", ] [[package]] name = "uu_uname" -version = "0.0.25" +version = "0.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5951832d73199636bde6c0d61cf960932b3c4450142c290375bc10c7abed6db5" +checksum = "ad1ca90f9b292bccaad0de70e6feccac5182c6713a5e1ca72d97bf3555b608b4" dependencies = [ "clap", "platform-info", - "uucore 0.0.26", + "uucore", ] [[package]] name = "uu_whoami" -version = "0.0.25" +version = "0.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b44166eb6335aeac42744ea368cc4c32d3f2287a4ff765a5ce44d927ab8bb4" +checksum = "bc7c52e42e0425710461700adc1063f468f2ba8a8ff83ee69ba661095ab7b77a" dependencies = [ "clap", "libc", - "uucore 0.0.26", + "uucore", "windows-sys 0.48.0", ] [[package]] name = "uucore" -version = "0.0.25" +version = "0.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23994a722acb43dbc56877e271c9723f167ae42c4c089f909b2d7dd106c3a9b4" -dependencies = [ - "clap", - "glob", - "libc", - "nix", - "once_cell", - "os_display", - "uucore_procs", - "wild", -] - -[[package]] -name = "uucore" -version = "0.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb2ea2f77699e5ff5c7e001af588ceb34cae8b5f9af5496bea5a6476aaa8e780" +checksum = "7b54aad02cf7e96f5fafabb6b836efa73eef934783b17530095a29ffd4fdc154" dependencies = [ "clap", "dunce", @@ -6506,9 +6627,9 @@ checksum = "425a23c7b7145bc7620c9c445817c37b1f78b6790aee9f208133f3c028975b60" [[package]] name = "uuid" -version = "1.8.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" dependencies = [ "getrandom", "serde", @@ -6751,6 +6872,20 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "webpage" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70862efc041d46e6bbaa82bb9c34ae0596d090e86cbd14bd9e93b36ee6802eac" +dependencies = [ + "curl", + "html5ever 0.27.0", + "markup5ever_rcdom", + "serde", + "serde_json", + "url", +] + [[package]] name = "which" version = "6.0.1" @@ -7163,6 +7298,17 @@ dependencies = [ "rustix", ] +[[package]] +name = "xml5ever" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bbb26405d8e919bc1547a5aa9abc95cbfa438f04844f5fdd9dc7596b748bf69" +dependencies = [ + "log", + "mac", + "markup5ever 0.12.1", +] + [[package]] name = "xxhash-rust" version = "0.8.10" diff --git a/Cargo.toml b/Cargo.toml index 2e9c7e0b0f..eb8a92c630 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ license = "MIT" name = "nu" repository = "https://github.com/nushell/nushell" rust-version = "1.77.2" -version = "0.93.1" +version = "0.95.1" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -32,7 +32,6 @@ members = [ "crates/nu-cmd-extra", "crates/nu-cmd-lang", "crates/nu-cmd-plugin", - "crates/nu-cmd-dataframe", "crates/nu-command", "crates/nu-color-config", "crates/nu-explore", @@ -40,6 +39,7 @@ members = [ "crates/nu-lsp", "crates/nu-pretty-hex", "crates/nu-protocol", + "crates/nu-derive-value", "crates/nu-plugin", "crates/nu-plugin-core", "crates/nu-plugin-engine", @@ -75,10 +75,12 @@ chardetng = "0.1.17" chrono = { default-features = false, version = "0.4.34" } chrono-humanize = "0.2.3" chrono-tz = "0.8" +convert_case = "0.6" crossbeam-channel = "0.5.8" crossterm = "0.27" csv = "1.3" ctrlc = "3.4" +deunicode = "1.6.0" dialoguer = { default-features = false, version = "0.11" } digest = { default-features = false, version = "0.10" } dirs-next = "2.0" @@ -94,7 +96,7 @@ heck = "0.5.0" human-date-parser = "0.1.1" indexmap = "2.2" indicatif = "0.17" -interprocess = "2.1.0" +interprocess = "2.2.0" is_executable = "1.0" itertools = "0.12" libc = "0.2" @@ -118,17 +120,20 @@ num-format = "0.4" num-traits = "0.2" omnipath = "0.1" once_cell = "1.18" -open = "5.1" -os_pipe = { version = "1.1", features = ["io_safety"] } +open = "5.2" +os_pipe = { version = "1.2", features = ["io_safety"] } pathdiff = "0.2" percent-encoding = "2" pretty_assertions = "1.4" print-positions = "0.6" +proc-macro-error = { version = "1.0", default-features = false } +proc-macro2 = "1.0" procfs = "0.16.0" pwd = "1.3" quick-xml = "0.31.0" quickcheck = "1.0" quickcheck_macros = "1.0" +quote = "1.0" rand = "0.8" ratatui = "0.26" rayon = "1.10" @@ -148,6 +153,7 @@ serde_urlencoded = "0.7.1" serde_yaml = "0.9" sha2 = "0.10" strip-ansi-escapes = "0.2.0" +syn = "2.0" sysinfo = "0.30" tabled = { version = "0.14.0", default-features = false } tempfile = "3.10" @@ -160,14 +166,14 @@ unicode-segmentation = "1.11" unicode-width = "0.1" ureq = { version = "2.9", default-features = false } url = "2.2" -uu_cp = "0.0.25" -uu_mkdir = "0.0.25" -uu_mktemp = "0.0.25" -uu_mv = "0.0.25" -uu_whoami = "0.0.25" -uu_uname = "0.0.25" -uucore = "0.0.25" -uuid = "1.8.0" +uu_cp = "0.0.27" +uu_mkdir = "0.0.27" +uu_mktemp = "0.0.27" +uu_mv = "0.0.27" +uu_whoami = "0.0.27" +uu_uname = "0.0.27" +uucore = "0.0.27" +uuid = "1.9.1" v_htmlescape = "0.15.0" wax = "0.6" which = "6.0.0" @@ -175,33 +181,30 @@ windows = "0.54" winreg = "0.52" [dependencies] -nu-cli = { path = "./crates/nu-cli", version = "0.93.1" } -nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.93.1" } -nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.93.1" } -nu-cmd-plugin = { path = "./crates/nu-cmd-plugin", version = "0.93.1", optional = true } -nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.93.1", features = [ - "dataframe", -], optional = true } -nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.93.1" } -nu-command = { path = "./crates/nu-command", version = "0.93.1" } -nu-engine = { path = "./crates/nu-engine", version = "0.93.1" } -nu-explore = { path = "./crates/nu-explore", version = "0.93.1" } -nu-lsp = { path = "./crates/nu-lsp/", version = "0.93.1" } -nu-parser = { path = "./crates/nu-parser", version = "0.93.1" } -nu-path = { path = "./crates/nu-path", version = "0.93.1" } -nu-plugin-engine = { path = "./crates/nu-plugin-engine", optional = true, version = "0.93.1" } -nu-protocol = { path = "./crates/nu-protocol", version = "0.93.1" } -nu-std = { path = "./crates/nu-std", version = "0.93.1" } -nu-system = { path = "./crates/nu-system", version = "0.93.1" } -nu-utils = { path = "./crates/nu-utils", version = "0.93.1" } - +nu-cli = { path = "./crates/nu-cli", version = "0.95.1" } +nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.95.1" } +nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.95.1" } +nu-cmd-plugin = { path = "./crates/nu-cmd-plugin", version = "0.95.1", optional = true } +nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.95.1" } +nu-command = { path = "./crates/nu-command", version = "0.95.1" } +nu-engine = { path = "./crates/nu-engine", version = "0.95.1" } +nu-explore = { path = "./crates/nu-explore", version = "0.95.1" } +nu-lsp = { path = "./crates/nu-lsp/", version = "0.95.1" } +nu-parser = { path = "./crates/nu-parser", version = "0.95.1" } +nu-path = { path = "./crates/nu-path", version = "0.95.1" } +nu-plugin-engine = { path = "./crates/nu-plugin-engine", optional = true, version = "0.95.1" } +nu-protocol = { path = "./crates/nu-protocol", version = "0.95.1" } +nu-std = { path = "./crates/nu-std", version = "0.95.1" } +nu-system = { path = "./crates/nu-system", version = "0.95.1" } +nu-utils = { path = "./crates/nu-utils", version = "0.95.1" } reedline = { workspace = true, features = ["bashisms", "sqlite"] } crossterm = { workspace = true } ctrlc = { workspace = true } +dirs-next = { workspace = true } log = { workspace = true } miette = { workspace = true, features = ["fancy-no-backtrace", "fancy"] } -mimalloc = { version = "0.1.37", default-features = false, optional = true } +mimalloc = { version = "0.1.42", default-features = false, optional = true } serde_json = { workspace = true } simplelog = "0.12" time = "0.3" @@ -222,13 +225,14 @@ nix = { workspace = true, default-features = false, features = [ ] } [dev-dependencies] -nu-test-support = { path = "./crates/nu-test-support", version = "0.93.1" } -nu-plugin-protocol = { path = "./crates/nu-plugin-protocol", version = "0.93.1" } -nu-plugin-core = { path = "./crates/nu-plugin-core", version = "0.93.1" } +nu-test-support = { path = "./crates/nu-test-support", version = "0.95.1" } +nu-plugin-protocol = { path = "./crates/nu-plugin-protocol", version = "0.95.1" } +nu-plugin-core = { path = "./crates/nu-plugin-core", version = "0.95.1" } assert_cmd = "2.0" dirs-next = { workspace = true } tango-bench = "0.5" pretty_assertions = { workspace = true } +regex = { workspace = true } rstest = { workspace = true, default-features = false } serial_test = "3.1" tempfile = { workspace = true } @@ -248,7 +252,6 @@ default = ["default-no-clipboard", "system-clipboard"] # See https://github.com/nushell/nushell/pull/11535 default-no-clipboard = [ "plugin", - "which-support", "trash-support", "sqlite", "mimalloc", @@ -268,12 +271,8 @@ system-clipboard = [ ] # Stable (Default) -which-support = ["nu-command/which-support", "nu-cmd-lang/which-support"] trash-support = ["nu-command/trash-support", "nu-cmd-lang/trash-support"] -# Dataframe feature for nushell -dataframe = ["dep:nu-cmd-dataframe", "nu-cmd-lang/dataframe"] - # SQLite commands for nushell sqlite = ["nu-command/sqlite", "nu-cmd-lang/sqlite"] diff --git a/README.md b/README.md index 0bfb7fc0c3..9b88a92528 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ To use `Nu` in GitHub Action, check [setup-nu](https://github.com/marketplace/ac Detailed installation instructions can be found in the [installation chapter of the book](https://www.nushell.sh/book/installation.html). Nu is available via many package managers: -[![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg)](https://repology.org/project/nushell/versions) +[![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg?columns=3)](https://repology.org/project/nushell/versions) For details about which platforms the Nushell team actively supports, see [our platform support policy](devdocs/PLATFORM_SUPPORT.md). @@ -222,6 +222,7 @@ Please submit an issue or PR to be added to this list. - [clap](https://github.com/clap-rs/clap/tree/master/clap_complete_nushell) - [Dorothy](http://github.com/bevry/dorothy) - [Direnv](https://github.com/direnv/direnv/blob/master/docs/hook.md#nushell) +- [x-cmd](https://x-cmd.com/mod/nu) ## Contributing diff --git a/benches/benchmarks.rs b/benches/benchmarks.rs index 84552daef9..4efb666507 100644 --- a/benches/benchmarks.rs +++ b/benches/benchmarks.rs @@ -4,11 +4,14 @@ use nu_plugin_protocol::{PluginCallResponse, PluginOutput}; use nu_protocol::{ engine::{EngineState, Stack}, - PipelineData, Span, Spanned, Value, + PipelineData, Signals, Span, Spanned, Value, }; use nu_std::load_standard_library; use nu_utils::{get_default_config, get_default_env}; -use std::rc::Rc; +use std::{ + rc::Rc, + sync::{atomic::AtomicBool, Arc}, +}; use std::hint::black_box; @@ -42,13 +45,16 @@ fn setup_stack_and_engine_from_command(command: &str) -> (Stack, EngineState) { }; let mut stack = Stack::new(); + + // Support running benchmarks with IR mode + stack.use_ir = std::env::var_os("NU_USE_IR").is_some(); + evaluate_commands( &commands, &mut engine, &mut stack, PipelineData::empty(), - None, - false, + Default::default(), ) .unwrap(); @@ -90,8 +96,7 @@ fn bench_command( &mut engine, &mut stack, PipelineData::empty(), - None, - false, + Default::default(), ) .unwrap(), ); @@ -250,14 +255,12 @@ fn bench_eval_interleave(n: i32) -> impl IntoBenchmarks { ) } -fn bench_eval_interleave_with_ctrlc(n: i32) -> impl IntoBenchmarks { +fn bench_eval_interleave_with_interrupt(n: i32) -> impl IntoBenchmarks { let mut engine = setup_engine(); - engine.ctrlc = Some(std::sync::Arc::new(std::sync::atomic::AtomicBool::new( - false, - ))); + engine.set_signals(Signals::new(Arc::new(AtomicBool::new(false)))); let stack = Stack::new(); bench_command( - &format!("eval_interleave_with_ctrlc_{n}"), + &format!("eval_interleave_with_interrupt_{n}"), &format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"), stack, engine, @@ -445,9 +448,9 @@ tango_benchmarks!( bench_eval_interleave(100), bench_eval_interleave(1_000), bench_eval_interleave(10_000), - bench_eval_interleave_with_ctrlc(100), - bench_eval_interleave_with_ctrlc(1_000), - bench_eval_interleave_with_ctrlc(10_000), + bench_eval_interleave_with_interrupt(100), + bench_eval_interleave_with_interrupt(1_000), + bench_eval_interleave_with_interrupt(10_000), // For bench_eval_for(1), bench_eval_for(10), diff --git a/crates/nu-cli/Cargo.toml b/crates/nu-cli/Cargo.toml index e631f7ccf6..2ed504d502 100644 --- a/crates/nu-cli/Cargo.toml +++ b/crates/nu-cli/Cargo.toml @@ -5,27 +5,27 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cli" edition = "2021" license = "MIT" name = "nu-cli" -version = "0.93.1" +version = "0.95.1" [lib] bench = false [dev-dependencies] -nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.1" } -nu-command = { path = "../nu-command", version = "0.93.1" } -nu-test-support = { path = "../nu-test-support", version = "0.93.1" } +nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.95.1" } +nu-command = { path = "../nu-command", version = "0.95.1" } +nu-test-support = { path = "../nu-test-support", version = "0.95.1" } rstest = { workspace = true, default-features = false } tempfile = { workspace = true } [dependencies] -nu-cmd-base = { path = "../nu-cmd-base", version = "0.93.1" } -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-path = { path = "../nu-path", version = "0.93.1" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.93.1", optional = true } -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } -nu-utils = { path = "../nu-utils", version = "0.93.1" } -nu-color-config = { path = "../nu-color-config", version = "0.93.1" } +nu-cmd-base = { path = "../nu-cmd-base", version = "0.95.1" } +nu-engine = { path = "../nu-engine", version = "0.95.1" } +nu-path = { path = "../nu-path", version = "0.95.1" } +nu-parser = { path = "../nu-parser", version = "0.95.1" } +nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.95.1", optional = true } +nu-protocol = { path = "../nu-protocol", version = "0.95.1" } +nu-utils = { path = "../nu-utils", version = "0.95.1" } +nu-color-config = { path = "../nu-color-config", version = "0.95.1" } nu-ansi-term = { workspace = true } reedline = { workspace = true, features = ["bashisms", "sqlite"] } @@ -39,7 +39,6 @@ miette = { workspace = true, features = ["fancy-no-backtrace"] } lscolors = { workspace = true, default-features = false, features = ["nu-ansi-term"] } once_cell = { workspace = true } percent-encoding = { workspace = true } -pathdiff = { workspace = true } sysinfo = { workspace = true } unicode-segmentation = { workspace = true } uuid = { workspace = true, features = ["v4"] } @@ -47,4 +46,4 @@ which = { workspace = true } [features] plugin = ["nu-plugin-engine"] -system-clipboard = ["reedline/system_clipboard"] +system-clipboard = ["reedline/system_clipboard"] \ No newline at end of file diff --git a/crates/nu-cli/src/commands/history/history_.rs b/crates/nu-cli/src/commands/history/history_.rs index 8b0714216e..cdf85eea72 100644 --- a/crates/nu-cli/src/commands/history/history_.rs +++ b/crates/nu-cli/src/commands/history/history_.rs @@ -47,7 +47,7 @@ impl Command for History { if let Some(config_path) = nu_path::config_dir() { let clear = call.has_flag(engine_state, stack, "clear")?; let long = call.has_flag(engine_state, stack, "long")?; - let ctrlc = engine_state.ctrlc.clone(); + let signals = engine_state.signals().clone(); let mut history_path = config_path; history_path.push("nushell"); @@ -107,7 +107,7 @@ impl Command for History { file: history_path.display().to_string(), span: head, })? - .into_pipeline_data(head, ctrlc)), + .into_pipeline_data(head, signals)), HistoryFileFormat::Sqlite => Ok(history_reader .and_then(|h| { h.search(SearchQuery::everything(SearchDirection::Forward, None)) @@ -122,7 +122,7 @@ impl Command for History { file: history_path.display().to_string(), span: head, })? - .into_pipeline_data(head, ctrlc)), + .into_pipeline_data(head, signals)), } } } else { diff --git a/crates/nu-cli/src/commands/keybindings_list.rs b/crates/nu-cli/src/commands/keybindings_list.rs index f4450c0c23..350df7b820 100644 --- a/crates/nu-cli/src/commands/keybindings_list.rs +++ b/crates/nu-cli/src/commands/keybindings_list.rs @@ -49,22 +49,24 @@ impl Command for KeybindingsList { fn run( &self, - _engine_state: &EngineState, - _stack: &mut Stack, + engine_state: &EngineState, + stack: &mut Stack, call: &Call, _input: PipelineData, ) -> Result { - let records = if call.named_len() == 0 { - let all_options = ["modifiers", "keycodes", "edits", "modes", "events"]; - all_options - .iter() - .flat_map(|argument| get_records(argument, call.head)) - .collect() - } else { - call.named_iter() - .flat_map(|(argument, _, _)| get_records(argument.item.as_str(), call.head)) - .collect() - }; + let all_options = ["modifiers", "keycodes", "edits", "modes", "events"]; + + let presence = all_options + .iter() + .map(|option| call.has_flag(engine_state, stack, option)) + .collect::, ShellError>>()?; + + let records = all_options + .iter() + .zip(presence) + .filter(|(_, present)| *present) + .flat_map(|(option, _)| get_records(option, call.head)) + .collect(); Ok(Value::list(records, call.head).into_pipeline_data()) } diff --git a/crates/nu-cli/src/completions/base.rs b/crates/nu-cli/src/completions/base.rs index 0debabe688..cf13dae68c 100644 --- a/crates/nu-cli/src/completions/base.rs +++ b/crates/nu-cli/src/completions/base.rs @@ -1,13 +1,12 @@ -use crate::completions::{CompletionOptions, SortBy}; +use crate::completions::CompletionOptions; use nu_protocol::{ engine::{Stack, StateWorkingSet}, - levenshtein_distance, Span, + Span, }; use reedline::Suggestion; -// Completer trait represents the three stages of the completion -// fetch, filter and sort pub trait Completer { + /// Fetch, filter, and sort completions #[allow(clippy::too_many_arguments)] fn fetch( &mut self, @@ -19,32 +18,6 @@ pub trait Completer { pos: usize, options: &CompletionOptions, ) -> Vec; - - fn get_sort_by(&self) -> SortBy { - SortBy::Ascending - } - - fn sort(&self, items: Vec, prefix: Vec) -> Vec { - let prefix_str = String::from_utf8_lossy(&prefix).to_string(); - let mut filtered_items = items; - - // Sort items - match self.get_sort_by() { - SortBy::LevenshteinDistance => { - filtered_items.sort_by(|a, b| { - let a_distance = levenshtein_distance(&prefix_str, &a.suggestion.value); - let b_distance = levenshtein_distance(&prefix_str, &b.suggestion.value); - a_distance.cmp(&b_distance) - }); - } - SortBy::Ascending => { - filtered_items.sort_by(|a, b| a.suggestion.value.cmp(&b.suggestion.value)); - } - SortBy::None => {} - }; - - filtered_items - } } #[derive(Debug, Default, PartialEq)] diff --git a/crates/nu-cli/src/completions/command_completions.rs b/crates/nu-cli/src/completions/command_completions.rs index 2549854540..e37f8b2576 100644 --- a/crates/nu-cli/src/completions/command_completions.rs +++ b/crates/nu-cli/src/completions/command_completions.rs @@ -9,7 +9,7 @@ use nu_protocol::{ }; use reedline::Suggestion; -use super::SemanticSuggestion; +use super::{completion_common::sort_suggestions, SemanticSuggestion}; pub struct CommandCompletion { flattened: Vec<(Span, FlatShape)>, @@ -161,7 +161,7 @@ impl Completer for CommandCompletion { &mut self, working_set: &StateWorkingSet, _stack: &Stack, - _prefix: Vec, + prefix: Vec, span: Span, offset: usize, pos: usize, @@ -198,7 +198,11 @@ impl Completer for CommandCompletion { }; if !subcommands.is_empty() { - return subcommands; + return sort_suggestions( + &String::from_utf8_lossy(&prefix), + subcommands, + SortBy::LevenshteinDistance, + ); } let config = working_set.get_config(); @@ -223,11 +227,11 @@ impl Completer for CommandCompletion { vec![] }; - subcommands.into_iter().chain(commands).collect::>() - } - - fn get_sort_by(&self) -> SortBy { - SortBy::LevenshteinDistance + sort_suggestions( + &String::from_utf8_lossy(&prefix), + commands, + SortBy::LevenshteinDistance, + ) } } diff --git a/crates/nu-cli/src/completions/completer.rs b/crates/nu-cli/src/completions/completer.rs index 007a0e288a..78398b4d19 100644 --- a/crates/nu-cli/src/completions/completer.rs +++ b/crates/nu-cli/src/completions/completer.rs @@ -51,8 +51,7 @@ impl NuCompleter { ..Default::default() }; - // Fetch - let mut suggestions = completer.fetch( + completer.fetch( working_set, &self.stack, prefix.clone(), @@ -60,12 +59,7 @@ impl NuCompleter { offset, pos, &options, - ); - - // Sort - suggestions = completer.sort(suggestions, prefix); - - suggestions + ) } fn external_completion( diff --git a/crates/nu-cli/src/completions/completion_common.rs b/crates/nu-cli/src/completions/completion_common.rs index 4a8b0d57ac..b3bca778a9 100644 --- a/crates/nu-cli/src/completions/completion_common.rs +++ b/crates/nu-cli/src/completions/completion_common.rs @@ -1,16 +1,21 @@ -use crate::completions::{matches, CompletionOptions}; +use crate::{ + completions::{matches, CompletionOptions}, + SemanticSuggestion, +}; use nu_ansi_term::Style; use nu_engine::env_to_string; -use nu_path::home_dir; +use nu_path::{expand_to_real_path, home_dir}; use nu_protocol::{ engine::{EngineState, Stack, StateWorkingSet}, - Span, + levenshtein_distance, Span, }; use nu_utils::get_ls_colors; use std::path::{ is_separator, Component, Path, PathBuf, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR, }; +use super::SortBy; + #[derive(Clone, Default)] pub struct PathBuiltFromString { parts: Vec, @@ -45,6 +50,7 @@ fn complete_rec( return completions; }; + let mut entries = Vec::new(); for entry in result.filter_map(|e| e.ok()) { let entry_name = entry.file_name().to_string_lossy().into_owned(); let entry_isdir = entry.path().is_dir(); @@ -53,20 +59,26 @@ fn complete_rec( built.isdir = entry_isdir; if !dir || entry_isdir { - match partial.split_first() { - Some((base, rest)) => { - if matches(base, &entry_name, options) { - if !rest.is_empty() || isdir { - completions - .extend(complete_rec(rest, &built, cwd, options, dir, isdir)); - } else { - completions.push(built); - } + entries.push((entry_name, built)); + } + } + + let prefix = partial.first().unwrap_or(&""); + let sorted_entries = sort_completions(prefix, entries, SortBy::Ascending, |(entry, _)| entry); + + for (entry_name, built) in sorted_entries { + match partial.split_first() { + Some((base, rest)) => { + if matches(base, &entry_name, options) { + if !rest.is_empty() || isdir { + completions.extend(complete_rec(rest, &built, cwd, options, dir, isdir)); + } else { + completions.push(built); } } - None => { - completions.push(built); - } + } + None => { + completions.push(built); } } } @@ -185,9 +197,14 @@ pub fn complete_item( .map(|p| { let path = original_cwd.apply(p); let style = ls_colors.as_ref().map(|lsc| { - lsc.style_for_path_with_metadata(&path, std::fs::symlink_metadata(&path).ok().as_ref()) - .map(lscolors::Style::to_nu_ansi_term_style) - .unwrap_or_default() + lsc.style_for_path_with_metadata( + &path, + std::fs::symlink_metadata(expand_to_real_path(&path)) + .ok() + .as_ref(), + ) + .map(lscolors::Style::to_nu_ansi_term_style) + .unwrap_or_default() }); (span, escape_path(path, want_directory), style) }) @@ -251,3 +268,38 @@ pub fn adjust_if_intermediate( readjusted, } } + +/// Convenience function to sort suggestions using [`sort_completions`] +pub fn sort_suggestions( + prefix: &str, + items: Vec, + sort_by: SortBy, +) -> Vec { + sort_completions(prefix, items, sort_by, |it| &it.suggestion.value) +} + +/// # Arguments +/// * `prefix` - What the user's typed, for sorting by Levenshtein distance +pub fn sort_completions( + prefix: &str, + mut items: Vec, + sort_by: SortBy, + get_value: fn(&T) -> &str, +) -> Vec { + // Sort items + match sort_by { + SortBy::LevenshteinDistance => { + items.sort_by(|a, b| { + let a_distance = levenshtein_distance(prefix, get_value(a)); + let b_distance = levenshtein_distance(prefix, get_value(b)); + a_distance.cmp(&b_distance) + }); + } + SortBy::Ascending => { + items.sort_by(|a, b| get_value(a).cmp(get_value(b))); + } + SortBy::None => {} + }; + + items +} diff --git a/crates/nu-cli/src/completions/custom_completions.rs b/crates/nu-cli/src/completions/custom_completions.rs index 17c8e6a924..0d2c674ef9 100644 --- a/crates/nu-cli/src/completions/custom_completions.rs +++ b/crates/nu-cli/src/completions/custom_completions.rs @@ -12,6 +12,8 @@ use nu_protocol::{ use nu_utils::IgnoreCaseExt; use std::collections::HashMap; +use super::completion_common::sort_suggestions; + pub struct CustomCompletion { stack: Stack, decl_id: usize, @@ -52,18 +54,16 @@ impl Completer for CustomCompletion { decl_id: self.decl_id, head: span, arguments: vec![ - Argument::Positional(Expression { - span: Span::unknown(), - ty: Type::String, - expr: Expr::String(self.line.clone()), - custom_completion: None, - }), - Argument::Positional(Expression { - span: Span::unknown(), - ty: Type::Int, - expr: Expr::Int(line_pos as i64), - custom_completion: None, - }), + Argument::Positional(Expression::new_unknown( + Expr::String(self.line.clone()), + Span::unknown(), + Type::String, + )), + Argument::Positional(Expression::new_unknown( + Expr::Int(line_pos as i64), + Span::unknown(), + Type::Int, + )), ], parser_info: HashMap::new(), }, @@ -124,15 +124,12 @@ impl Completer for CustomCompletion { }) .unwrap_or_default(); - if let Some(custom_completion_options) = custom_completion_options { + let suggestions = if let Some(custom_completion_options) = custom_completion_options { filter(&prefix, suggestions, &custom_completion_options) } else { filter(&prefix, suggestions, completion_options) - } - } - - fn get_sort_by(&self) -> SortBy { - self.sort_by + }; + sort_suggestions(&String::from_utf8_lossy(&prefix), suggestions, self.sort_by) } } diff --git a/crates/nu-cli/src/completions/directory_completions.rs b/crates/nu-cli/src/completions/directory_completions.rs index 024322f997..61b0439444 100644 --- a/crates/nu-cli/src/completions/directory_completions.rs +++ b/crates/nu-cli/src/completions/directory_completions.rs @@ -1,14 +1,14 @@ use crate::completions::{ completion_common::{adjust_if_intermediate, complete_item, AdjustView}, - Completer, CompletionOptions, SortBy, + Completer, CompletionOptions, }; use nu_ansi_term::Style; use nu_protocol::{ engine::{EngineState, Stack, StateWorkingSet}, - levenshtein_distance, Span, + Span, }; use reedline::Suggestion; -use std::path::{Path, MAIN_SEPARATOR as SEP}; +use std::path::Path; use super::SemanticSuggestion; @@ -36,7 +36,7 @@ impl Completer for DirectoryCompletion { // Filter only the folders #[allow(deprecated)] - let output: Vec<_> = directory_completion( + let items: Vec<_> = directory_completion( span, &prefix, &working_set.permanent_state.current_work_dir(), @@ -62,41 +62,11 @@ impl Completer for DirectoryCompletion { }) .collect(); - output - } - - // Sort results prioritizing the non hidden folders - fn sort(&self, items: Vec, prefix: Vec) -> Vec { - let prefix_str = String::from_utf8_lossy(&prefix).to_string(); - - // Sort items - let mut sorted_items = items; - - match self.get_sort_by() { - SortBy::Ascending => { - sorted_items.sort_by(|a, b| { - // Ignore trailing slashes in folder names when sorting - a.suggestion - .value - .trim_end_matches(SEP) - .cmp(b.suggestion.value.trim_end_matches(SEP)) - }); - } - SortBy::LevenshteinDistance => { - sorted_items.sort_by(|a, b| { - let a_distance = levenshtein_distance(&prefix_str, &a.suggestion.value); - let b_distance = levenshtein_distance(&prefix_str, &b.suggestion.value); - a_distance.cmp(&b_distance) - }); - } - _ => (), - } - // Separate the results between hidden and non hidden let mut hidden: Vec = vec![]; let mut non_hidden: Vec = vec![]; - for item in sorted_items.into_iter() { + for item in items.into_iter() { let item_path = Path::new(&item.suggestion.value); if let Some(value) = item_path.file_name() { diff --git a/crates/nu-cli/src/completions/dotnu_completions.rs b/crates/nu-cli/src/completions/dotnu_completions.rs index c939578b41..1b2bbab20c 100644 --- a/crates/nu-cli/src/completions/dotnu_completions.rs +++ b/crates/nu-cli/src/completions/dotnu_completions.rs @@ -1,4 +1,4 @@ -use crate::completions::{file_path_completion, Completer, CompletionOptions, SortBy}; +use crate::completions::{file_path_completion, Completer, CompletionOptions}; use nu_protocol::{ engine::{Stack, StateWorkingSet}, Span, @@ -6,7 +6,7 @@ use nu_protocol::{ use reedline::Suggestion; use std::path::{is_separator, Path, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR}; -use super::SemanticSuggestion; +use super::{completion_common::sort_suggestions, SemanticSuggestion, SortBy}; #[derive(Clone, Default)] pub struct DotNuCompletion {} @@ -131,10 +131,6 @@ impl Completer for DotNuCompletion { }) .collect(); - output - } - - fn get_sort_by(&self) -> SortBy { - SortBy::LevenshteinDistance + sort_suggestions(&prefix_str, output, SortBy::Ascending) } } diff --git a/crates/nu-cli/src/completions/file_completions.rs b/crates/nu-cli/src/completions/file_completions.rs index f6205f6792..10a548e7ab 100644 --- a/crates/nu-cli/src/completions/file_completions.rs +++ b/crates/nu-cli/src/completions/file_completions.rs @@ -1,15 +1,15 @@ use crate::completions::{ completion_common::{adjust_if_intermediate, complete_item, AdjustView}, - Completer, CompletionOptions, SortBy, + Completer, CompletionOptions, }; use nu_ansi_term::Style; use nu_protocol::{ engine::{EngineState, Stack, StateWorkingSet}, - levenshtein_distance, Span, + Span, }; use nu_utils::IgnoreCaseExt; use reedline::Suggestion; -use std::path::{Path, MAIN_SEPARATOR as SEP}; +use std::path::Path; use super::SemanticSuggestion; @@ -40,7 +40,7 @@ impl Completer for FileCompletion { } = adjust_if_intermediate(&prefix, working_set, span); #[allow(deprecated)] - let output: Vec<_> = complete_item( + let items: Vec<_> = complete_item( readjusted, span, &prefix, @@ -67,41 +67,13 @@ impl Completer for FileCompletion { }) .collect(); - output - } - - // Sort results prioritizing the non hidden folders - fn sort(&self, items: Vec, prefix: Vec) -> Vec { - let prefix_str = String::from_utf8_lossy(&prefix).to_string(); - - // Sort items - let mut sorted_items = items; - - match self.get_sort_by() { - SortBy::Ascending => { - sorted_items.sort_by(|a, b| { - // Ignore trailing slashes in folder names when sorting - a.suggestion - .value - .trim_end_matches(SEP) - .cmp(b.suggestion.value.trim_end_matches(SEP)) - }); - } - SortBy::LevenshteinDistance => { - sorted_items.sort_by(|a, b| { - let a_distance = levenshtein_distance(&prefix_str, &a.suggestion.value); - let b_distance = levenshtein_distance(&prefix_str, &b.suggestion.value); - a_distance.cmp(&b_distance) - }); - } - _ => (), - } + // Sort results prioritizing the non hidden folders // Separate the results between hidden and non hidden let mut hidden: Vec = vec![]; let mut non_hidden: Vec = vec![]; - for item in sorted_items.into_iter() { + for item in items.into_iter() { let item_path = Path::new(&item.suggestion.value); if let Some(value) = item_path.file_name() { diff --git a/crates/nu-cli/src/completions/flag_completions.rs b/crates/nu-cli/src/completions/flag_completions.rs index b0dcc0963b..ea4ddd6856 100644 --- a/crates/nu-cli/src/completions/flag_completions.rs +++ b/crates/nu-cli/src/completions/flag_completions.rs @@ -1,4 +1,6 @@ -use crate::completions::{Completer, CompletionOptions}; +use crate::completions::{ + completion_common::sort_suggestions, Completer, CompletionOptions, SortBy, +}; use nu_protocol::{ ast::{Expr, Expression}, engine::{Stack, StateWorkingSet}, @@ -90,7 +92,7 @@ impl Completer for FlagCompletion { } } - return output; + return sort_suggestions(&String::from_utf8_lossy(&prefix), output, SortBy::Ascending); } vec![] diff --git a/crates/nu-cli/src/completions/variable_completions.rs b/crates/nu-cli/src/completions/variable_completions.rs index 0572fe93c1..72a69e942c 100644 --- a/crates/nu-cli/src/completions/variable_completions.rs +++ b/crates/nu-cli/src/completions/variable_completions.rs @@ -9,6 +9,8 @@ use nu_protocol::{ use reedline::Suggestion; use std::str; +use super::{completion_common::sort_suggestions, SortBy}; + #[derive(Clone)] pub struct VariableCompletion { var_context: (Vec, Vec>), // tuple with $var and the sublevels (.b.c.d) @@ -40,6 +42,7 @@ impl Completer for VariableCompletion { end: span.end - offset, }; let sublevels_count = self.var_context.1.len(); + let prefix_str = String::from_utf8_lossy(&prefix); // Completions for the given variable if !var_str.is_empty() { @@ -69,7 +72,7 @@ impl Completer for VariableCompletion { } } - return output; + return sort_suggestions(&prefix_str, output, SortBy::Ascending); } } else { // No nesting provided, return all env vars @@ -93,7 +96,7 @@ impl Completer for VariableCompletion { } } - return output; + return sort_suggestions(&prefix_str, output, SortBy::Ascending); } } @@ -117,7 +120,7 @@ impl Completer for VariableCompletion { } } - return output; + return sort_suggestions(&prefix_str, output, SortBy::Ascending); } } @@ -139,7 +142,7 @@ impl Completer for VariableCompletion { } } - return output; + return sort_suggestions(&prefix_str, output, SortBy::Ascending); } } } @@ -226,6 +229,8 @@ impl Completer for VariableCompletion { } } + output = sort_suggestions(&prefix_str, output, SortBy::Ascending); + output.dedup(); // TODO: Removes only consecutive duplicates, is it intended? output diff --git a/crates/nu-cli/src/config_files.rs b/crates/nu-cli/src/config_files.rs index ec7ad2f412..3a02f75e86 100644 --- a/crates/nu-cli/src/config_files.rs +++ b/crates/nu-cli/src/config_files.rs @@ -8,7 +8,7 @@ use nu_protocol::{ report_error_new, HistoryFileFormat, PipelineData, }; #[cfg(feature = "plugin")] -use nu_utils::utils::perf; +use nu_utils::perf; use std::path::PathBuf; #[cfg(feature = "plugin")] @@ -53,13 +53,10 @@ pub fn read_plugin_file( // Reading signatures from plugin registry file // The plugin.msgpackz file stores the parsed signature collected from each registered plugin add_plugin_file(engine_state, plugin_file.clone(), storage_path); - perf( + perf!( "add plugin file to engine_state", start_time, - file!(), - line!(), - column!(), - engine_state.get_config().use_ansi_coloring, + engine_state.get_config().use_ansi_coloring ); start_time = std::time::Instant::now(); @@ -137,13 +134,10 @@ pub fn read_plugin_file( } }; - perf( + perf!( &format!("read plugin file {}", plugin_path.display()), start_time, - file!(), - line!(), - column!(), - engine_state.get_config().use_ansi_coloring, + engine_state.get_config().use_ansi_coloring ); start_time = std::time::Instant::now(); @@ -156,13 +150,10 @@ pub fn read_plugin_file( return; } - perf( + perf!( &format!("load plugin file {}", plugin_path.display()), start_time, - file!(), - line!(), - column!(), - engine_state.get_config().use_ansi_coloring, + engine_state.get_config().use_ansi_coloring ); } } @@ -344,7 +335,10 @@ pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) - name: identity.name().to_owned(), filename: identity.filename().to_owned(), shell: identity.shell().map(|p| p.to_owned()), - data: PluginRegistryItemData::Valid { commands }, + data: PluginRegistryItemData::Valid { + metadata: Default::default(), + commands, + }, }); } @@ -378,13 +372,10 @@ pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) - ); } - perf( + perf!( "migrate old plugin file", start_time, - file!(), - line!(), - column!(), - engine_state.get_config().use_ansi_coloring, + engine_state.get_config().use_ansi_coloring ); true } diff --git a/crates/nu-cli/src/eval_cmds.rs b/crates/nu-cli/src/eval_cmds.rs index 8fa3bf30e5..ad3a15304d 100644 --- a/crates/nu-cli/src/eval_cmds.rs +++ b/crates/nu-cli/src/eval_cmds.rs @@ -8,15 +8,45 @@ use nu_protocol::{ }; use std::sync::Arc; +#[derive(Default)] +pub struct EvaluateCommandsOpts { + pub table_mode: Option, + pub error_style: Option, + pub no_newline: bool, +} + /// Run a command (or commands) given to us by the user pub fn evaluate_commands( commands: &Spanned, engine_state: &mut EngineState, stack: &mut Stack, input: PipelineData, - table_mode: Option, - no_newline: bool, + opts: EvaluateCommandsOpts, ) -> Result<(), ShellError> { + let EvaluateCommandsOpts { + table_mode, + error_style, + no_newline, + } = opts; + + // Handle the configured error style early + if let Some(e_style) = error_style { + match e_style.coerce_str()?.parse() { + Ok(e_style) => { + Arc::make_mut(&mut engine_state.config).error_style = e_style; + } + Err(err) => { + return Err(ShellError::GenericError { + error: "Invalid value for `--error-style`".into(), + msg: err.into(), + span: Some(e_style.span()), + help: None, + inner: vec![], + }); + } + } + } + // Translate environment variables from Strings to Values convert_env_values(engine_state, stack)?; @@ -40,6 +70,11 @@ pub fn evaluate_commands( std::process::exit(1); } + if let Some(err) = working_set.compile_errors.first() { + report_error(&working_set, err); + // Not a fatal error, for now + } + (output, working_set.render()) }; diff --git a/crates/nu-cli/src/lib.rs b/crates/nu-cli/src/lib.rs index c4342dc3a0..6f151adad1 100644 --- a/crates/nu-cli/src/lib.rs +++ b/crates/nu-cli/src/lib.rs @@ -17,7 +17,7 @@ mod validation; pub use commands::add_cli_context; pub use completions::{FileCompletion, NuCompleter, SemanticSuggestion, SuggestionKind}; pub use config_files::eval_config_contents; -pub use eval_cmds::evaluate_commands; +pub use eval_cmds::{evaluate_commands, EvaluateCommandsOpts}; pub use eval_file::evaluate_file; pub use menus::NuHelpCompleter; pub use nu_cmd_base::util::get_init_cwd; diff --git a/crates/nu-cli/src/nu_highlight.rs b/crates/nu-cli/src/nu_highlight.rs index 07084c6258..f4f38296de 100644 --- a/crates/nu-cli/src/nu_highlight.rs +++ b/crates/nu-cli/src/nu_highlight.rs @@ -32,7 +32,7 @@ impl Command for NuHighlight { ) -> Result { let head = call.head; - let ctrlc = engine_state.ctrlc.clone(); + let signals = engine_state.signals(); let engine_state = std::sync::Arc::new(engine_state.clone()); let config = engine_state.get_config().clone(); @@ -50,7 +50,7 @@ impl Command for NuHighlight { } Err(err) => Value::error(err, head), }, - ctrlc, + signals, ) } diff --git a/crates/nu-cli/src/reedline_config.rs b/crates/nu-cli/src/reedline_config.rs index 3f920a00cb..dd5a3199dc 100644 --- a/crates/nu-cli/src/reedline_config.rs +++ b/crates/nu-cli/src/reedline_config.rs @@ -1,6 +1,5 @@ use crate::{menus::NuMenuCompleter, NuHelpCompleter}; use crossterm::event::{KeyCode, KeyModifiers}; -use log::trace; use nu_ansi_term::Style; use nu_color_config::{color_record_to_nustyle, lookup_ansi_color_style}; use nu_engine::eval_block; @@ -76,15 +75,15 @@ const DEFAULT_HELP_MENU: &str = r#" // Adds all menus to line editor pub(crate) fn add_menus( mut line_editor: Reedline, - engine_state: Arc, + engine_state_ref: Arc, stack: &Stack, config: &Config, ) -> Result { - trace!("add_menus: config: {:#?}", &config); + //log::trace!("add_menus: config: {:#?}", &config); line_editor = line_editor.clear_menus(); for menu in &config.menus { - line_editor = add_menu(line_editor, menu, engine_state.clone(), stack, config)? + line_editor = add_menu(line_editor, menu, engine_state_ref.clone(), stack, config)? } // Checking if the default menus have been added from the config file @@ -94,13 +93,16 @@ pub(crate) fn add_menus( ("help_menu", DEFAULT_HELP_MENU), ]; + let mut engine_state = (*engine_state_ref).clone(); + let mut menu_eval_results = vec![]; + for (name, definition) in default_menus { if !config .menus .iter() .any(|menu| menu.name.to_expanded_string("", config) == name) { - let (block, _) = { + let (block, delta) = { let mut working_set = StateWorkingSet::new(&engine_state); let output = parse( &mut working_set, @@ -112,15 +114,31 @@ pub(crate) fn add_menus( (output, working_set.render()) }; + engine_state.merge_delta(delta)?; + let mut temp_stack = Stack::new().capture(); let input = PipelineData::Empty; - let res = eval_block::(&engine_state, &mut temp_stack, &block, input)?; + menu_eval_results.push(eval_block::( + &engine_state, + &mut temp_stack, + &block, + input, + )?); + } + } - if let PipelineData::Value(value, None) = res { - for menu in create_menus(&value)? { - line_editor = - add_menu(line_editor, &menu, engine_state.clone(), stack, config)?; - } + let new_engine_state_ref = Arc::new(engine_state); + + for res in menu_eval_results.into_iter() { + if let PipelineData::Value(value, None) = res { + for menu in create_menus(&value)? { + line_editor = add_menu( + line_editor, + &menu, + new_engine_state_ref.clone(), + stack, + config, + )?; } } } diff --git a/crates/nu-cli/src/repl.rs b/crates/nu-cli/src/repl.rs index 29c2f62734..07272701f3 100644 --- a/crates/nu-cli/src/repl.rs +++ b/crates/nu-cli/src/repl.rs @@ -31,7 +31,7 @@ use nu_protocol::{ }; use nu_utils::{ filesystem::{have_permission, PermissionResult}, - utils::perf, + perf, }; use reedline::{ CursorConfig, CwdAwareHinter, DefaultCompleter, EditCommand, Emacs, FileBackedHistory, @@ -43,7 +43,7 @@ use std::{ io::{self, IsTerminal, Write}, panic::{catch_unwind, AssertUnwindSafe}, path::{Path, PathBuf}, - sync::{atomic::Ordering, Arc}, + sync::Arc, time::{Duration, Instant}, }; use sysinfo::System; @@ -89,14 +89,7 @@ pub fn evaluate_repl( if let Err(e) = convert_env_values(engine_state, &unique_stack) { report_error_new(engine_state, &e); } - perf( - "translate env vars", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("translate env vars", start_time, use_color); // seed env vars unique_stack.add_env_var( @@ -225,28 +218,14 @@ fn get_line_editor( // Now that reedline is created, get the history session id and store it in engine_state store_history_id_in_engine(engine_state, &line_editor); - perf( - "setup reedline", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("setup reedline", start_time, use_color); if let Some(history) = engine_state.history_config() { start_time = std::time::Instant::now(); line_editor = setup_history(nushell_path, engine_state, line_editor, history)?; - perf( - "setup history", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("setup history", start_time, use_color); } Ok(line_editor) } @@ -289,28 +268,14 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { if let Err(err) = engine_state.merge_env(&mut stack, cwd) { report_error_new(engine_state, &err); } - perf( - "merge env", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + // Check whether $env.NU_USE_IR is set, so that the user can change it in the REPL + // Temporary while IR eval is optional + stack.use_ir = stack.has_env_var(engine_state, "NU_USE_IR"); + perf!("merge env", start_time, use_color); start_time = std::time::Instant::now(); - // Reset the ctrl-c handler - if let Some(ctrlc) = &mut engine_state.ctrlc { - ctrlc.store(false, Ordering::SeqCst); - } - perf( - "reset ctrlc", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + engine_state.reset_signals(); + perf!("reset signals", start_time, use_color); start_time = std::time::Instant::now(); // Right before we start our prompt and take input from the user, @@ -320,14 +285,7 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { report_error_new(engine_state, &err); } } - perf( - "pre-prompt hook", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("pre-prompt hook", start_time, use_color); start_time = std::time::Instant::now(); // Next, check all the environment variables they ask for @@ -336,14 +294,7 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { if let Err(error) = hook::eval_env_change_hook(env_change, engine_state, &mut stack) { report_error_new(engine_state, &error) } - perf( - "env-change hook", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("env-change hook", start_time, use_color); let engine_reference = Arc::new(engine_state.clone()); let config = engine_state.get_config(); @@ -355,14 +306,7 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { vi_normal: map_nucursorshape_to_cursorshape(config.cursor_shape_vi_normal), emacs: map_nucursorshape_to_cursorshape(config.cursor_shape_emacs), }; - perf( - "get config/cursor config", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("get config/cursor config", start_time, use_color); start_time = std::time::Instant::now(); // at this line we have cloned the state for the completer and the transient prompt @@ -392,16 +336,17 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { .with_quick_completions(config.quick_completions) .with_partial_completions(config.partial_completions) .with_ansi_colors(config.use_ansi_coloring) + .with_cwd(Some( + engine_state + .cwd(None) + .map(|cwd| cwd.into_std_path_buf()) + .unwrap_or_default() + .to_string_lossy() + .to_string(), + )) .with_cursor_config(cursor_config); - perf( - "reedline builder", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("reedline builder", start_time, use_color); let style_computer = StyleComputer::from_config(engine_state, &stack_arc); @@ -416,14 +361,7 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { line_editor.disable_hints() }; - perf( - "reedline coloring/style_computer", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("reedline coloring/style_computer", start_time, use_color); start_time = std::time::Instant::now(); trace!("adding menus"); @@ -433,14 +371,7 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { Reedline::create() }); - perf( - "reedline adding menus", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("reedline adding menus", start_time, use_color); start_time = std::time::Instant::now(); let buffer_editor = get_editor(engine_state, &stack_arc, Span::unknown()); @@ -457,14 +388,7 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { line_editor }; - perf( - "reedline buffer_editor", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("reedline buffer_editor", start_time, use_color); if let Some(history) = engine_state.history_config() { start_time = std::time::Instant::now(); @@ -474,28 +398,14 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { } } - perf( - "sync_history", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("sync_history", start_time, use_color); } start_time = std::time::Instant::now(); // Changing the line editor based on the found keybindings line_editor = setup_keybindings(engine_state, line_editor); - perf( - "keybindings", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("keybindings", start_time, use_color); start_time = std::time::Instant::now(); let config = &engine_state.get_config().clone(); @@ -512,14 +422,7 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { nu_prompt, ); - perf( - "update_prompt", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("update_prompt", start_time, use_color); *entry_num += 1; @@ -546,14 +449,7 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { // so we should avoid it or making stack cheaper to clone. let mut stack = Arc::unwrap_or_clone(stack_arc); - perf( - "line_editor setup", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("line_editor setup", start_time, use_color); let line_editor_input_time = std::time::Instant::now(); match input { @@ -590,14 +486,7 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { } } - perf( - "pre_execution_hook", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("pre_execution_hook", start_time, use_color); let mut repl = engine_state.repl_state.lock().expect("repl state mutex"); repl.cursor_pos = line_editor.current_insertion_point(); @@ -612,26 +501,20 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { run_ansi_sequence(VSCODE_PRE_EXECUTION_MARKER); - perf( + perf!( "pre_execute_marker (633;C) ansi escape sequence", start_time, - file!(), - line!(), - column!(), - use_color, + use_color ); } else if shell_integration_osc133 { start_time = Instant::now(); run_ansi_sequence(PRE_EXECUTION_MARKER); - perf( + perf!( "pre_execute_marker (133;C) ansi escape sequence", start_time, - file!(), - line!(), - column!(), - use_color, + use_color ); } } else if shell_integration_osc133 { @@ -639,13 +522,10 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { run_ansi_sequence(PRE_EXECUTION_MARKER); - perf( + perf!( "pre_execute_marker (133;C) ansi escape sequence", start_time, - file!(), - line!(), - column!(), - use_color, + use_color ); } @@ -769,22 +649,16 @@ fn loop_iteration(ctx: LoopContext) -> (bool, Stack, Reedline) { ); } } - perf( + perf!( "processing line editor input", line_editor_input_time, - file!(), - line!(), - column!(), - use_color, + use_color ); - perf( + perf!( "time between prompts in line editor loop", loop_start_time, - file!(), - line!(), - column!(), - use_color, + use_color ); (true, stack, line_editor) @@ -800,13 +674,14 @@ fn prepare_history_metadata( line_editor: &mut Reedline, ) { if !s.is_empty() && line_editor.has_last_command_context() { - #[allow(deprecated)] let result = line_editor .update_last_command_context(&|mut c| { c.start_timestamp = Some(chrono::Utc::now()); c.hostname = hostname.map(str::to_string); - - c.cwd = Some(StateWorkingSet::new(engine_state).get_cwd()); + c.cwd = engine_state + .cwd(None) + .ok() + .map(|path| path.to_string_lossy().to_string()); c }) .into_diagnostic(); @@ -1061,14 +936,7 @@ fn run_shell_integration_osc2( // ESC]2;stringBEL -- Set window title to string run_ansi_sequence(&format!("\x1b]2;{title}\x07")); - perf( - "set title with command osc2", - start_time, - file!(), - line!(), - column!(), - use_color, - ); + perf!("set title with command osc2", start_time, use_color); } } @@ -1093,13 +961,10 @@ fn run_shell_integration_osc7( percent_encoding::utf8_percent_encode(&path, percent_encoding::CONTROLS) )); - perf( + perf!( "communicate path to terminal with osc7", start_time, - file!(), - line!(), - column!(), - use_color, + use_color ); } } @@ -1110,19 +975,16 @@ fn run_shell_integration_osc9_9(engine_state: &EngineState, stack: &mut Stack, u let start_time = Instant::now(); // Otherwise, communicate the path as OSC 9;9 from ConEmu (often used for spawning new tabs in the same dir) + // This is helpful in Windows Terminal with Duplicate Tab run_ansi_sequence(&format!( - "\x1b]9;9;{}{}\x1b\\", - if path.starts_with('/') { "" } else { "/" }, + "\x1b]9;9;{}\x1b\\", percent_encoding::utf8_percent_encode(&path, percent_encoding::CONTROLS) )); - perf( + perf!( "communicate path to terminal with osc9;9", start_time, - file!(), - line!(), - column!(), - use_color, + use_color ); } } @@ -1142,13 +1004,10 @@ fn run_shell_integration_osc633(engine_state: &EngineState, stack: &mut Stack, u VSCODE_CWD_PROPERTY_MARKER_PREFIX, path, VSCODE_CWD_PROPERTY_MARKER_SUFFIX )); - perf( + perf!( "communicate path to terminal with osc633;P", start_time, - file!(), - line!(), - column!(), - use_color, + use_color ); } } @@ -1371,13 +1230,10 @@ fn run_finaliziation_ansi_sequence( shell_integration_osc133, )); - perf( + perf!( "post_execute_marker (633;D) ansi escape sequences", start_time, - file!(), - line!(), - column!(), - use_color, + use_color ); } else if shell_integration_osc133 { let start_time = Instant::now(); @@ -1389,13 +1245,10 @@ fn run_finaliziation_ansi_sequence( shell_integration_osc133, )); - perf( + perf!( "post_execute_marker (133;D) ansi escape sequences", start_time, - file!(), - line!(), - column!(), - use_color, + use_color ); } } else if shell_integration_osc133 { @@ -1408,13 +1261,10 @@ fn run_finaliziation_ansi_sequence( shell_integration_osc133, )); - perf( + perf!( "post_execute_marker (133;D) ansi escape sequences", start_time, - file!(), - line!(), - column!(), - use_color, + use_color ); } } diff --git a/crates/nu-cli/src/syntax_highlight.rs b/crates/nu-cli/src/syntax_highlight.rs index e296943af6..41ef168390 100644 --- a/crates/nu-cli/src/syntax_highlight.rs +++ b/crates/nu-cli/src/syntax_highlight.rs @@ -138,6 +138,7 @@ impl Highlighter for NuHighlighter { FlatShape::Filepath => add_colored_token(&shape.1, next_token), FlatShape::Directory => add_colored_token(&shape.1, next_token), + FlatShape::GlobInterpolation => add_colored_token(&shape.1, next_token), FlatShape::GlobPattern => add_colored_token(&shape.1, next_token), FlatShape::Variable(_) | FlatShape::VarDecl(_) => { add_colored_token(&shape.1, next_token) @@ -452,15 +453,17 @@ fn find_matching_block_end_in_expr( } } - Expr::StringInterpolation(exprs) => exprs.iter().find_map(|expr| { - find_matching_block_end_in_expr( - line, - working_set, - expr, - global_span_offset, - global_cursor_offset, - ) - }), + Expr::StringInterpolation(exprs) | Expr::GlobInterpolation(exprs, _) => { + exprs.iter().find_map(|expr| { + find_matching_block_end_in_expr( + line, + working_set, + expr, + global_span_offset, + global_cursor_offset, + ) + }) + } Expr::List(list) => { if expr_last == global_cursor_offset { diff --git a/crates/nu-cli/src/util.rs b/crates/nu-cli/src/util.rs index 7ebea0deb2..bcee53c9b0 100644 --- a/crates/nu-cli/src/util.rs +++ b/crates/nu-cli/src/util.rs @@ -8,7 +8,7 @@ use nu_protocol::{ }; #[cfg(windows)] use nu_utils::enable_vt_processing; -use nu_utils::utils::perf; +use nu_utils::perf; use std::path::Path; // This will collect environment variables from std::env and adds them to a stack. @@ -228,13 +228,10 @@ pub fn eval_source( let _ = enable_vt_processing(); } - perf( + perf!( &format!("eval_source {}", &fname), start_time, - file!(), - line!(), - column!(), - engine_state.get_config().use_ansi_coloring, + engine_state.get_config().use_ansi_coloring ); exit_code @@ -265,6 +262,11 @@ fn evaluate_source( return Ok(Some(1)); } + if let Some(err) = working_set.compile_errors.first() { + report_error(&working_set, err); + // Not a fatal error, for now + } + (output, working_set.render()) }; @@ -276,8 +278,8 @@ fn evaluate_source( eval_block::(engine_state, stack, &block, input) }?; - let status = if let PipelineData::ByteStream(stream, ..) = pipeline { - stream.print(false)? + let status = if let PipelineData::ByteStream(..) = pipeline { + pipeline.print(engine_state, stack, false, false)? } else { if let Some(hook) = engine_state.get_config().hooks.display_output.clone() { let pipeline = eval_hook( diff --git a/crates/nu-cli/tests/completions/mod.rs b/crates/nu-cli/tests/completions/mod.rs index cb883b67db..a7f88dc3b0 100644 --- a/crates/nu-cli/tests/completions/mod.rs +++ b/crates/nu-cli/tests/completions/mod.rs @@ -11,7 +11,7 @@ use std::{ sync::Arc, }; use support::{ - completions_helpers::{new_partial_engine, new_quote_engine}, + completions_helpers::{new_dotnu_engine, new_partial_engine, new_quote_engine}, file, folder, match_suggestions, new_engine, }; @@ -85,8 +85,29 @@ fn custom_completer() -> NuCompleter { NuCompleter::new(Arc::new(engine), Arc::new(stack)) } +#[fixture] +fn subcommand_completer() -> NuCompleter { + // Create a new engine + let (dir, _, mut engine, mut stack) = new_engine(); + + // Use fuzzy matching, because subcommands are sorted by Levenshtein distance, + // and that's not very useful with prefix matching + let commands = r#" + $env.config.completions.algorithm = "fuzzy" + def foo [] {} + def "foo bar" [] {} + def "foo abaz" [] {} + def "foo aabrr" [] {} + def food [] {} + "#; + assert!(support::merge_input(commands.as_bytes(), &mut engine, &mut stack, dir).is_ok()); + + // Instantiate a new completer + NuCompleter::new(Arc::new(engine), Arc::new(stack)) +} + #[test] -fn variables_dollar_sign_with_varialblecompletion() { +fn variables_dollar_sign_with_variablecompletion() { let (_, _, engine, stack) = new_engine(); let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); @@ -138,43 +159,42 @@ fn variables_customcompletion_subcommands_with_customcompletion_2( #[test] fn dotnu_completions() { // Create a new engine - let (_, _, engine, stack) = new_engine(); + let (_, _, engine, stack) = new_dotnu_engine(); // Instantiate a new completer let mut completer = NuCompleter::new(Arc::new(engine), Arc::new(stack)); + let expected = vec![ + "asdf.nu".into(), + "bar.nu".into(), + "bat.nu".into(), + "baz.nu".into(), + #[cfg(windows)] + "dir_module\\".into(), + #[cfg(not(windows))] + "dir_module/".into(), + "foo.nu".into(), + "spam.nu".into(), + "xyzzy.nu".into(), + ]; + // Test source completion let completion_str = "source-env ".to_string(); let suggestions = completer.complete(&completion_str, completion_str.len()); - assert_eq!(2, suggestions.len()); - assert_eq!("custom_completion.nu", suggestions.first().unwrap().value); - #[cfg(windows)] - assert_eq!("directory_completion\\", suggestions.get(1).unwrap().value); - #[cfg(not(windows))] - assert_eq!("directory_completion/", suggestions.get(1).unwrap().value); + match_suggestions(expected.clone(), suggestions); // Test use completion let completion_str = "use ".to_string(); let suggestions = completer.complete(&completion_str, completion_str.len()); - assert_eq!(2, suggestions.len()); - assert_eq!("custom_completion.nu", suggestions.first().unwrap().value); - #[cfg(windows)] - assert_eq!("directory_completion\\", suggestions.get(1).unwrap().value); - #[cfg(not(windows))] - assert_eq!("directory_completion/", suggestions.get(1).unwrap().value); + match_suggestions(expected.clone(), suggestions); // Test overlay use completion let completion_str = "overlay use ".to_string(); let suggestions = completer.complete(&completion_str, completion_str.len()); - assert_eq!(2, suggestions.len()); - assert_eq!("custom_completion.nu", suggestions.first().unwrap().value); - #[cfg(windows)] - assert_eq!("directory_completion\\", suggestions.get(1).unwrap().value); - #[cfg(not(windows))] - assert_eq!("directory_completion/", suggestions.get(1).unwrap().value); + match_suggestions(expected, suggestions); } #[test] @@ -276,9 +296,10 @@ fn partial_completions() { // Create the expected values let expected_paths: Vec = vec![ - folder(dir.join("partial_a")), - folder(dir.join("partial_b")), - folder(dir.join("partial_c")), + folder(dir.join("partial")), + folder(dir.join("partial-a")), + folder(dir.join("partial-b")), + folder(dir.join("partial-c")), ]; // Match the results @@ -292,11 +313,14 @@ fn partial_completions() { // Create the expected values let expected_paths: Vec = vec![ - file(dir.join("partial_a").join("hello")), - file(dir.join("partial_a").join("hola")), - file(dir.join("partial_b").join("hello_b")), - file(dir.join("partial_b").join("hi_b")), - file(dir.join("partial_c").join("hello_c")), + file(dir.join("partial").join("hello.txt")), + file(dir.join("partial-a").join("have_ext.exe")), + file(dir.join("partial-a").join("have_ext.txt")), + file(dir.join("partial-a").join("hello")), + file(dir.join("partial-a").join("hola")), + file(dir.join("partial-b").join("hello_b")), + file(dir.join("partial-b").join("hi_b")), + file(dir.join("partial-c").join("hello_c")), ]; // Match the results @@ -309,12 +333,15 @@ fn partial_completions() { // Create the expected values let expected_paths: Vec = vec![ - file(dir.join("partial_a").join("anotherfile")), - file(dir.join("partial_a").join("hello")), - file(dir.join("partial_a").join("hola")), - file(dir.join("partial_b").join("hello_b")), - file(dir.join("partial_b").join("hi_b")), - file(dir.join("partial_c").join("hello_c")), + file(dir.join("partial").join("hello.txt")), + file(dir.join("partial-a").join("anotherfile")), + file(dir.join("partial-a").join("have_ext.exe")), + file(dir.join("partial-a").join("have_ext.txt")), + file(dir.join("partial-a").join("hello")), + file(dir.join("partial-a").join("hola")), + file(dir.join("partial-b").join("hello_b")), + file(dir.join("partial-b").join("hi_b")), + file(dir.join("partial-c").join("hello_c")), ]; // Match the results @@ -339,23 +366,57 @@ fn partial_completions() { // Create the expected values let expected_paths: Vec = vec![ file( - dir.join("partial_a") + dir.join("partial") .join("..") .join("final_partial") .join("somefile"), ), file( - dir.join("partial_b") + dir.join("partial-a") .join("..") .join("final_partial") .join("somefile"), ), file( - dir.join("partial_c") + dir.join("partial-b") .join("..") .join("final_partial") .join("somefile"), ), + file( + dir.join("partial-c") + .join("..") + .join("final_partial") + .join("somefile"), + ), + ]; + + // Match the results + match_suggestions(expected_paths, suggestions); + + // Test completion for all files under directories whose names begin with "pa" + let file_str = file(dir.join("partial-a").join("have")); + let target_file = format!("rm {file_str}"); + let suggestions = completer.complete(&target_file, target_file.len()); + + // Create the expected values + let expected_paths: Vec = vec![ + file(dir.join("partial-a").join("have_ext.exe")), + file(dir.join("partial-a").join("have_ext.txt")), + ]; + + // Match the results + match_suggestions(expected_paths, suggestions); + + // Test completion for all files under directories whose names begin with "pa" + let file_str = file(dir.join("partial-a").join("have_ext.")); + let file_dir = format!("rm {file_str}"); + let suggestions = completer.complete(&file_dir, file_dir.len()); + + // Create the expected values + let expected_paths: Vec = vec![ + file(dir.join("partial-a").join("have_ext.exe")), + file(dir.join("partial-a").join("have_ext.txt")), ]; // Match the results @@ -394,6 +455,13 @@ fn command_ls_with_filecompletion() { ".hidden_folder/".to_string(), ]; + match_suggestions(expected_paths, suggestions); + + let target_dir = "ls custom_completion."; + let suggestions = completer.complete(target_dir, target_dir.len()); + + let expected_paths: Vec = vec!["custom_completion.nu".to_string()]; + match_suggestions(expected_paths, suggestions) } #[test] @@ -428,6 +496,13 @@ fn command_open_with_filecompletion() { ".hidden_folder/".to_string(), ]; + match_suggestions(expected_paths, suggestions); + + let target_dir = "open custom_completion."; + let suggestions = completer.complete(target_dir, target_dir.len()); + + let expected_paths: Vec = vec!["custom_completion.nu".to_string()]; + match_suggestions(expected_paths, suggestions) } @@ -606,6 +681,27 @@ fn command_watch_with_filecompletion() { match_suggestions(expected_paths, suggestions) } +#[rstest] +fn subcommand_completions(mut subcommand_completer: NuCompleter) { + let prefix = "foo br"; + let suggestions = subcommand_completer.complete(prefix, prefix.len()); + match_suggestions( + vec!["foo bar".to_string(), "foo aabrr".to_string()], + suggestions, + ); + + let prefix = "foo b"; + let suggestions = subcommand_completer.complete(prefix, prefix.len()); + match_suggestions( + vec![ + "foo bar".to_string(), + "foo abaz".to_string(), + "foo aabrr".to_string(), + ], + suggestions, + ); +} + #[test] fn file_completion_quoted() { let (_, _, engine, stack) = new_quote_engine(); @@ -616,11 +712,11 @@ fn file_completion_quoted() { let suggestions = completer.complete(target_dir, target_dir.len()); let expected_paths: Vec = vec![ - "\'[a] bc.txt\'".to_string(), "`--help`".to_string(), "`-42`".to_string(), "`-inf`".to_string(), "`4.2`".to_string(), + "\'[a] bc.txt\'".to_string(), "`te st.txt`".to_string(), "`te#st.txt`".to_string(), "`te'st.txt`".to_string(), @@ -717,11 +813,13 @@ fn variables_completions() { // Test completions for $nu let suggestions = completer.complete("$nu.", 4); - assert_eq!(15, suggestions.len()); + assert_eq!(18, suggestions.len()); let expected: Vec = vec![ + "cache-dir".into(), "config-path".into(), "current-exe".into(), + "data-dir".into(), "default-config-dir".into(), "env-path".into(), "history-enabled".into(), @@ -735,6 +833,7 @@ fn variables_completions() { "plugin-path".into(), "startup-time".into(), "temp-path".into(), + "vendor-autoload-dir".into(), ]; // Match results @@ -808,6 +907,11 @@ fn variables_completions() { // Match results match_suggestions(expected, suggestions); + + let suggestions = completer.complete("$", 1); + let expected: Vec = vec!["$actor".into(), "$env".into(), "$in".into(), "$nu".into()]; + + match_suggestions(expected, suggestions); } #[test] diff --git a/crates/nu-cli/tests/completions/support/completions_helpers.rs b/crates/nu-cli/tests/completions/support/completions_helpers.rs index 47f46ab00e..027ac9d997 100644 --- a/crates/nu-cli/tests/completions/support/completions_helpers.rs +++ b/crates/nu-cli/tests/completions/support/completions_helpers.rs @@ -68,6 +68,52 @@ pub fn new_engine() -> (PathBuf, String, EngineState, Stack) { (dir, dir_str, engine_state, stack) } +// creates a new engine with the current path into the completions fixtures folder +pub fn new_dotnu_engine() -> (PathBuf, String, EngineState, Stack) { + // Target folder inside assets + let dir = fs::fixtures().join("dotnu_completions"); + let dir_str = dir + .clone() + .into_os_string() + .into_string() + .unwrap_or_default(); + let dir_span = nu_protocol::Span::new(0, dir_str.len()); + + // Create a new engine with default context + let mut engine_state = create_default_context(); + + // Add $nu + engine_state.generate_nu_constant(); + + // New stack + let mut stack = Stack::new(); + + // Add pwd as env var + stack.add_env_var("PWD".to_string(), Value::string(dir_str.clone(), dir_span)); + stack.add_env_var( + "TEST".to_string(), + Value::string("NUSHELL".to_string(), dir_span), + ); + + stack.add_env_var( + "NU_LIB_DIRS".to_string(), + Value::List { + vals: vec![ + Value::string(file(dir.join("lib-dir1")), dir_span), + Value::string(file(dir.join("lib-dir2")), dir_span), + Value::string(file(dir.join("lib-dir3")), dir_span), + ], + internal_span: dir_span, + }, + ); + + // Merge environment into the permanent state + let merge_result = engine_state.merge_env(&mut stack, &dir); + assert!(merge_result.is_ok()); + + (dir, dir_str, engine_state, stack) +} + pub fn new_quote_engine() -> (PathBuf, String, EngineState, Stack) { // Target folder inside assets let dir = fs::fixtures().join("quoted_completions"); @@ -149,9 +195,13 @@ pub fn match_suggestions(expected: Vec, suggestions: Vec) { Expected: {expected:#?}\n" ) } - expected.iter().zip(suggestions).for_each(|it| { - assert_eq!(it.0, &it.1.value); - }); + assert_eq!( + expected, + suggestions + .into_iter() + .map(|it| it.value) + .collect::>() + ); } // append the separator to the converted path diff --git a/crates/nu-cmd-base/Cargo.toml b/crates/nu-cmd-base/Cargo.toml index b163ff447c..2fe8610f49 100644 --- a/crates/nu-cmd-base/Cargo.toml +++ b/crates/nu-cmd-base/Cargo.toml @@ -5,17 +5,17 @@ edition = "2021" license = "MIT" name = "nu-cmd-base" repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-base" -version = "0.93.1" +version = "0.95.1" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-path = { path = "../nu-path", version = "0.93.1" } -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } +nu-engine = { path = "../nu-engine", version = "0.95.1" } +nu-parser = { path = "../nu-parser", version = "0.95.1" } +nu-path = { path = "../nu-path", version = "0.95.1" } +nu-protocol = { path = "../nu-protocol", version = "0.95.1" } indexmap = { workspace = true } miette = { workspace = true } -[dev-dependencies] +[dev-dependencies] \ No newline at end of file diff --git a/crates/nu-cmd-base/src/hook.rs b/crates/nu-cmd-base/src/hook.rs index 76c13bd5c3..cef5348618 100644 --- a/crates/nu-cmd-base/src/hook.rs +++ b/crates/nu-cmd-base/src/hook.rs @@ -194,7 +194,7 @@ pub fn eval_hook( let Some(follow) = val.get("code") else { return Err(ShellError::CantFindColumn { col_name: "code".into(), - span, + span: Some(span), src_span: span, }); }; diff --git a/crates/nu-cmd-base/src/input_handler.rs b/crates/nu-cmd-base/src/input_handler.rs index d81193e190..7d61f90cb0 100644 --- a/crates/nu-cmd-base/src/input_handler.rs +++ b/crates/nu-cmd-base/src/input_handler.rs @@ -1,5 +1,5 @@ -use nu_protocol::{ast::CellPath, PipelineData, ShellError, Span, Value}; -use std::sync::{atomic::AtomicBool, Arc}; +use nu_protocol::{ast::CellPath, PipelineData, ShellError, Signals, Span, Value}; +use std::sync::Arc; pub trait CmdArgument { fn take_cell_paths(&mut self) -> Option>; @@ -40,7 +40,7 @@ pub fn operate( mut arg: A, input: PipelineData, span: Span, - ctrlc: Option>, + signals: &Signals, ) -> Result where A: CmdArgument + Send + Sync + 'static, @@ -55,7 +55,7 @@ where _ => cmd(&v, &arg, span), } }, - ctrlc, + signals, ), Some(column_paths) => { let arg = Arc::new(arg); @@ -79,7 +79,7 @@ where } v }, - ctrlc, + signals, ) } } diff --git a/crates/nu-cmd-base/src/util.rs b/crates/nu-cmd-base/src/util.rs index 9c63dec836..e485243877 100644 --- a/crates/nu-cmd-base/src/util.rs +++ b/crates/nu-cmd-base/src/util.rs @@ -1,3 +1,4 @@ +use nu_path::AbsolutePathBuf; use nu_protocol::{ engine::{EngineState, Stack}, Range, ShellError, Span, Value, @@ -15,11 +16,13 @@ pub fn get_init_cwd() -> PathBuf { pub fn get_guaranteed_cwd(engine_state: &EngineState, stack: &Stack) -> PathBuf { engine_state .cwd(Some(stack)) + .map(AbsolutePathBuf::into_std_path_buf) .unwrap_or(crate::util::get_init_cwd()) } type MakeRangeError = fn(&str, Span) -> ShellError; +/// Returns a inclusive pair of boundary in given `range`. pub fn process_range(range: &Range) -> Result<(isize, isize), MakeRangeError> { match range { Range::IntRange(range) => { diff --git a/crates/nu-cmd-dataframe/Cargo.toml b/crates/nu-cmd-dataframe/Cargo.toml deleted file mode 100644 index a156435a8d..0000000000 --- a/crates/nu-cmd-dataframe/Cargo.toml +++ /dev/null @@ -1,75 +0,0 @@ -[package] -authors = ["The Nushell Project Developers"] -description = "Nushell's dataframe commands based on polars." -edition = "2021" -license = "MIT" -name = "nu-cmd-dataframe" -repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-dataframe" -version = "0.93.1" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[lib] -bench = false - -[dependencies] -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } - -# Potential dependencies for extras -chrono = { workspace = true, features = ["std", "unstable-locales"], default-features = false } -chrono-tz = { workspace = true } -fancy-regex = { workspace = true } -indexmap = { workspace = true } -num = { version = "0.4", optional = true } -serde = { workspace = true, features = ["derive"] } -# keep sqlparser at 0.39.0 until we can update polars -sqlparser = { version = "0.45", optional = true } -polars-io = { version = "0.39", features = ["avro"], optional = true } -polars-arrow = { version = "0.39", optional = true } -polars-ops = { version = "0.39", optional = true } -polars-plan = { version = "0.39", features = ["regex"], optional = true } -polars-utils = { version = "0.39", optional = true } - -[dependencies.polars] -features = [ - "arg_where", - "checked_arithmetic", - "concat_str", - "cross_join", - "csv", - "cum_agg", - "dtype-categorical", - "dtype-datetime", - "dtype-struct", - "dtype-i8", - "dtype-i16", - "dtype-u8", - "dtype-u16", - "dynamic_group_by", - "ipc", - "is_in", - "json", - "lazy", - "object", - "parquet", - "random", - "rolling_window", - "rows", - "serde", - "serde-lazy", - "strings", - "temporal", - "to_dummies", -] -default-features = false -optional = true -version = "0.39" - -[features] -dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "polars-utils", "sqlparser"] -default = [] - -[dev-dependencies] -nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.1" } diff --git a/crates/nu-cmd-dataframe/src/dataframe/README.md b/crates/nu-cmd-dataframe/src/dataframe/README.md deleted file mode 100644 index 593217ede6..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Dataframe - -This dataframe directory holds all of the definitions of the dataframe data structures and commands. - -There are three sections of commands: - -* [eager](./eager) -* [series](./series) -* [values](./values) - -For more details see the -[Nushell book section on dataframes](https://www.nushell.sh/book/dataframes.html) diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/append.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/append.rs deleted file mode 100644 index c0be67ed6e..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/append.rs +++ /dev/null @@ -1,134 +0,0 @@ -use crate::dataframe::values::{Axis, Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct AppendDF; - -impl Command for AppendDF { - fn name(&self) -> &str { - "dfr append" - } - - fn usage(&self) -> &str { - "Appends a new dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("other", SyntaxShape::Any, "dataframe to be appended") - .switch("col", "appends in col orientation", Some('c')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Appends a dataframe as new columns", - example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df); - $a | dfr append $a"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - Column::new( - "a_x".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b_x".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Appends a dataframe merging at the end of columns", - example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df); - $a | dfr append $a --col"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![ - Value::test_int(1), - Value::test_int(3), - Value::test_int(1), - Value::test_int(3), - ], - ), - Column::new( - "b".to_string(), - vec![ - Value::test_int(2), - Value::test_int(4), - Value::test_int(2), - Value::test_int(4), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let other: Value = call.req(engine_state, stack, 0)?; - - let axis = if call.has_flag(engine_state, stack, "col")? { - Axis::Column - } else { - Axis::Row - }; - let df_other = NuDataFrame::try_from_value(other)?; - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - df.append_df(&df_other, axis, call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(AppendDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs deleted file mode 100644 index be9c33a229..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs +++ /dev/null @@ -1,195 +0,0 @@ -use crate::dataframe::values::{str_to_dtype, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::*; - -#[derive(Clone)] -pub struct CastDF; - -impl Command for CastDF { - fn name(&self) -> &str { - "dfr cast" - } - - fn usage(&self) -> &str { - "Cast a column to a different dtype." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .required( - "dtype", - SyntaxShape::String, - "The dtype to cast the column to", - ) - .optional( - "column", - SyntaxShape::String, - "The column to cast. Required when used with a dataframe.", - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Cast a column in a dataframe to a different dtype", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr cast u8 a | dfr schema", - result: Some(Value::record( - record! { - "a" => Value::string("u8", Span::test_data()), - "b" => Value::string("i64", Span::test_data()), - }, - Span::test_data(), - )), - }, - Example { - description: "Cast a column in a lazy dataframe to a different dtype", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr into-lazy | dfr cast u8 a | dfr schema", - result: Some(Value::record( - record! { - "a" => Value::string("u8", Span::test_data()), - "b" => Value::string("i64", Span::test_data()), - }, - Span::test_data(), - )), - }, - Example { - description: "Cast a column in a expression to a different dtype", - example: r#"[[a b]; [1 2] [1 4]] | dfr into-df | dfr group-by a | dfr agg [ (dfr col b | dfr cast u8 | dfr min | dfr as "b_min") ] | dfr schema"#, - result: None - } - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuLazyFrame::can_downcast(&value) { - let (dtype, column_nm) = df_args(engine_state, stack, call)?; - let df = NuLazyFrame::try_from_value(value)?; - command_lazy(call, column_nm, dtype, df) - } else if NuDataFrame::can_downcast(&value) { - let (dtype, column_nm) = df_args(engine_state, stack, call)?; - let df = NuDataFrame::try_from_value(value)?; - command_eager(call, column_nm, dtype, df) - } else { - let dtype: String = call.req(engine_state, stack, 0)?; - let dtype = str_to_dtype(&dtype, call.head)?; - - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().cast(dtype).into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } -} - -fn df_args( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result<(DataType, String), ShellError> { - let dtype = dtype_arg(engine_state, stack, call)?; - let column_nm: String = - call.opt(engine_state, stack, 1)? - .ok_or(ShellError::MissingParameter { - param_name: "column_name".into(), - span: call.head, - })?; - Ok((dtype, column_nm)) -} - -fn dtype_arg( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let dtype: String = call.req(engine_state, stack, 0)?; - str_to_dtype(&dtype, call.head) -} - -fn command_lazy( - call: &Call, - column_nm: String, - dtype: DataType, - lazy: NuLazyFrame, -) -> Result { - let column = col(&column_nm).cast(dtype); - let lazy = lazy.into_polars().with_columns(&[column]); - let lazy = NuLazyFrame::new(false, lazy); - - Ok(PipelineData::Value( - NuLazyFrame::into_value(lazy, call.head)?, - None, - )) -} - -fn command_eager( - call: &Call, - column_nm: String, - dtype: DataType, - nu_df: NuDataFrame, -) -> Result { - let mut df = nu_df.df; - let column = df - .column(&column_nm) - .map_err(|e| ShellError::GenericError { - error: format!("{e}"), - msg: "".into(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let casted = column.cast(&dtype).map_err(|e| ShellError::GenericError { - error: format!("{e}"), - msg: "".into(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let _ = df - .with_column(casted) - .map_err(|e| ShellError::GenericError { - error: format!("{e}"), - msg: "".into(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let df = NuDataFrame::new(false, df); - Ok(PipelineData::Value(df.into_value(call.head), None)) -} - -#[cfg(test)] -mod test { - - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(CastDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/columns.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/columns.rs deleted file mode 100644 index c9167659b5..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/columns.rs +++ /dev/null @@ -1,73 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ColumnsDF; - -impl Command for ColumnsDF { - fn name(&self) -> &str { - "dfr columns" - } - - fn usage(&self) -> &str { - "Show dataframe columns." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type(Type::Custom("dataframe".into()), Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Dataframe columns", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr columns", - result: Some(Value::list( - vec![Value::test_string("a"), Value::test_string("b")], - Span::test_data(), - )), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let names: Vec = df - .as_ref() - .get_column_names() - .iter() - .map(|v| Value::string(*v, call.head)) - .collect(); - - let names = Value::list(names, call.head); - - Ok(PipelineData::Value(names, None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ColumnsDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs deleted file mode 100644 index 8f9d086947..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs +++ /dev/null @@ -1,115 +0,0 @@ -use crate::dataframe::values::{utils::convert_columns, Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct DropDF; - -impl Command for DropDF { - fn name(&self) -> &str { - "dfr drop" - } - - fn usage(&self) -> &str { - "Creates a new dataframe by dropping the selected columns." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest("rest", SyntaxShape::Any, "column names to be dropped") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "drop column a", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr drop a", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let columns: Vec = call.rest(engine_state, stack, 0)?; - let (col_string, col_span) = convert_columns(columns, call.head)?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let new_df = col_string - .first() - .ok_or_else(|| ShellError::GenericError { - error: "Empty names list".into(), - msg: "No column names were found".into(), - span: Some(col_span), - help: None, - inner: vec![], - }) - .and_then(|col| { - df.as_ref() - .drop(&col.item) - .map_err(|e| ShellError::GenericError { - error: "Error dropping column".into(), - msg: e.to_string(), - span: Some(col.span), - help: None, - inner: vec![], - }) - })?; - - // If there are more columns in the drop selection list, these - // are added from the resulting dataframe - col_string - .iter() - .skip(1) - .try_fold(new_df, |new_df, col| { - new_df - .drop(&col.item) - .map_err(|e| ShellError::GenericError { - error: "Error dropping column".into(), - msg: e.to_string(), - span: Some(col.span), - help: None, - inner: vec![], - }) - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(DropDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs deleted file mode 100644 index b2ae6f7cfc..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs +++ /dev/null @@ -1,119 +0,0 @@ -use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::UniqueKeepStrategy; - -#[derive(Clone)] -pub struct DropDuplicates; - -impl Command for DropDuplicates { - fn name(&self) -> &str { - "dfr drop-duplicates" - } - - fn usage(&self) -> &str { - "Drops duplicate values in dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .optional( - "subset", - SyntaxShape::Table(vec![]), - "subset of columns to drop duplicates", - ) - .switch("maintain", "maintain order", Some('m')) - .switch( - "last", - "keeps last duplicate value (by default keeps first)", - Some('l'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "drop duplicates", - example: "[[a b]; [1 2] [3 4] [1 2]] | dfr into-df | dfr drop-duplicates", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(3), Value::test_int(1)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(4), Value::test_int(2)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let columns: Option> = call.opt(engine_state, stack, 0)?; - let (subset, col_span) = match columns { - Some(cols) => { - let (agg_string, col_span) = convert_columns_string(cols, call.head)?; - (Some(agg_string), col_span) - } - None => (None, call.head), - }; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let subset_slice = subset.as_ref().map(|cols| &cols[..]); - - let keep_strategy = if call.has_flag(engine_state, stack, "last")? { - UniqueKeepStrategy::Last - } else { - UniqueKeepStrategy::First - }; - - df.as_ref() - .unique(subset_slice, keep_strategy, None) - .map_err(|e| ShellError::GenericError { - error: "Error dropping duplicates".into(), - msg: e.to_string(), - span: Some(col_span), - help: None, - inner: vec![], - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(DropDuplicates {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs deleted file mode 100644 index 25a3907426..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs +++ /dev/null @@ -1,137 +0,0 @@ -use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct DropNulls; - -impl Command for DropNulls { - fn name(&self) -> &str { - "dfr drop-nulls" - } - - fn usage(&self) -> &str { - "Drops null values in dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .optional( - "subset", - SyntaxShape::Table(vec![]), - "subset of columns to drop nulls", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "drop null values in dataframe", - example: r#"let df = ([[a b]; [1 2] [3 0] [1 2]] | dfr into-df); - let res = ($df.b / $df.b); - let a = ($df | dfr with-column $res --name res); - $a | dfr drop-nulls"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(1)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(2)], - ), - Column::new( - "res".to_string(), - vec![Value::test_int(1), Value::test_int(1)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "drop null values in dataframe", - example: r#"let s = ([1 2 0 0 3 4] | dfr into-df); - ($s / $s) | dfr drop-nulls"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "div_0_0".to_string(), - vec![ - Value::test_int(1), - Value::test_int(1), - Value::test_int(1), - Value::test_int(1), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let columns: Option> = call.opt(engine_state, stack, 0)?; - - let (subset, col_span) = match columns { - Some(cols) => { - let (agg_string, col_span) = convert_columns_string(cols, call.head)?; - (Some(agg_string), col_span) - } - None => (None, call.head), - }; - - let subset_slice = subset.as_ref().map(|cols| &cols[..]); - - df.as_ref() - .drop_nulls(subset_slice) - .map_err(|e| ShellError::GenericError { - error: "Error dropping nulls".into(), - msg: e.to_string(), - span: Some(col_span), - help: None, - inner: vec![], - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::super::WithColumn; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(DropNulls {}), Box::new(WithColumn {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs deleted file mode 100644 index a572a49551..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs +++ /dev/null @@ -1,104 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct DataTypes; - -impl Command for DataTypes { - fn name(&self) -> &str { - "dfr dtypes" - } - - fn usage(&self) -> &str { - "Show dataframe data types." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Dataframe dtypes", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dtypes", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "column".to_string(), - vec![Value::test_string("a"), Value::test_string("b")], - ), - Column::new( - "dtype".to_string(), - vec![Value::test_string("i64"), Value::test_string("i64")], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut dtypes: Vec = Vec::new(); - let names: Vec = df - .as_ref() - .get_column_names() - .iter() - .map(|v| { - let dtype = df - .as_ref() - .column(v) - .expect("using name from list of names from dataframe") - .dtype(); - - let dtype_str = dtype.to_string(); - - dtypes.push(Value::string(dtype_str, call.head)); - - Value::string(*v, call.head) - }) - .collect(); - - let names_col = Column::new("column".to_string(), names); - let dtypes_col = Column::new("dtype".to_string(), dtypes); - - NuDataFrame::try_from_columns(vec![names_col, dtypes_col], None) - .map(|df| PipelineData::Value(df.into_value(call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(DataTypes {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs deleted file mode 100644 index f47f65a004..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs +++ /dev/null @@ -1,107 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::{prelude::*, series::Series}; - -#[derive(Clone)] -pub struct Dummies; - -impl Command for Dummies { - fn name(&self) -> &str { - "dfr dummies" - } - - fn usage(&self) -> &str { - "Creates a new dataframe with dummy variables." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .switch("drop-first", "Drop first row", Some('d')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create new dataframe with dummy variables from a dataframe", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dummies", - result: Some( - NuDataFrame::try_from_series( - vec![ - Series::new("a_1", &[1_u8, 0]), - Series::new("a_3", &[0_u8, 1]), - Series::new("b_2", &[1_u8, 0]), - Series::new("b_4", &[0_u8, 1]), - ], - Span::test_data(), - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Create new dataframe with dummy variables from a series", - example: "[1 2 2 3 3] | dfr into-df | dfr dummies", - result: Some( - NuDataFrame::try_from_series( - vec![ - Series::new("0_1", &[1_u8, 0, 0, 0, 0]), - Series::new("0_2", &[0_u8, 1, 1, 0, 0]), - Series::new("0_3", &[0_u8, 0, 0, 1, 1]), - ], - Span::test_data(), - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let drop_first: bool = call.has_flag(engine_state, stack, "drop-first")?; - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - df.as_ref() - .to_dummies(None, drop_first) - .map_err(|e| ShellError::GenericError { - error: "Error calculating dummies".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The only allowed column types for dummies are String or Int".into()), - inner: vec![], - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Dummies {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs deleted file mode 100644 index 14c86e8c40..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/first.rs +++ /dev/null @@ -1,144 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct FirstDF; - -impl Command for FirstDF { - fn name(&self) -> &str { - "dfr first" - } - - fn usage(&self) -> &str { - "Show only the first number of rows or create a first expression" - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .optional( - "rows", - SyntaxShape::Int, - "starting from the front, the number of rows to return", - ) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Return the first row of a dataframe", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_int(1)]), - Column::new("b".to_string(), vec![Value::test_int(2)]), - ], - None, - ) - .expect("should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Return the first two rows of a dataframe", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first 2", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - ], - None, - ) - .expect("should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates a first expression from a column", - example: "dfr col a | dfr first", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let df = NuDataFrame::try_from_value(value)?; - command(engine_state, stack, call, df) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().first().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let rows: Option = call.opt(engine_state, stack, 0)?; - let rows = rows.unwrap_or(1); - - let res = df.as_ref().head(Some(rows)); - Ok(PipelineData::Value( - NuDataFrame::dataframe_into_value(res, call.head), - None, - )) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example}; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(FirstDF {})]); - test_dataframe_example(&mut engine_state, &FirstDF.examples()[0]); - test_dataframe_example(&mut engine_state, &FirstDF.examples()[1]); - } - - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(FirstDF {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &FirstDF.examples()[2]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/get.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/get.rs deleted file mode 100644 index e8cf337864..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/get.rs +++ /dev/null @@ -1,87 +0,0 @@ -use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct GetDF; - -impl Command for GetDF { - fn name(&self) -> &str { - "dfr get" - } - - fn usage(&self) -> &str { - "Creates dataframe with the selected columns." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest("rest", SyntaxShape::Any, "column names to sort dataframe") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns the selected column", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr get a", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let columns: Vec = call.rest(engine_state, stack, 0)?; - let (col_string, col_span) = convert_columns_string(columns, call.head)?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - df.as_ref() - .select(col_string) - .map_err(|e| ShellError::GenericError { - error: "Error selecting columns".into(), - msg: e.to_string(), - span: Some(col_span), - help: None, - inner: vec![], - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs deleted file mode 100644 index ff2c4f98a2..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/last.rs +++ /dev/null @@ -1,118 +0,0 @@ -use crate::dataframe::values::{utils::DEFAULT_ROWS, Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LastDF; - -impl Command for LastDF { - fn name(&self) -> &str { - "dfr last" - } - - fn usage(&self) -> &str { - "Creates new dataframe with tail rows or creates a last expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .optional("rows", SyntaxShape::Int, "Number of rows for tail") - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create new dataframe with last rows", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr last 1", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_int(3)]), - Column::new("b".to_string(), vec![Value::test_int(4)]), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates a last expression from a column", - example: "dfr col a | dfr last", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let df = NuDataFrame::try_from_value(value)?; - command(engine_state, stack, call, df) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().last().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let rows: Option = call.opt(engine_state, stack, 0)?; - let rows = rows.unwrap_or(DEFAULT_ROWS); - - let res = df.as_ref().tail(Some(rows)); - Ok(PipelineData::Value( - NuDataFrame::dataframe_into_value(res, call.head), - None, - )) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example}; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(LastDF {})]); - test_dataframe_example(&mut engine_state, &LastDF.examples()[0]); - } - - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(LastDF {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &LastDF.examples()[1]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/list.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/list.rs deleted file mode 100644 index 1cee694180..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/list.rs +++ /dev/null @@ -1,68 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ListDF; - -impl Command for ListDF { - fn name(&self) -> &str { - "dfr ls" - } - - fn usage(&self) -> &str { - "Lists stored dataframes." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()).category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Creates a new dataframe and shows it in the dataframe list", - example: r#"let test = ([[a b];[1 2] [3 4]] | dfr into-df); - ls"#, - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - let mut vals: Vec<(String, Value)> = vec![]; - - for overlay_frame in engine_state.active_overlays(&[]) { - for var in &overlay_frame.vars { - if let Ok(value) = stack.get_var(*var.1, call.head) { - let name = String::from_utf8_lossy(var.0).to_string(); - vals.push((name, value)); - } - } - } - - let vals = vals - .into_iter() - .filter_map(|(name, value)| { - NuDataFrame::try_from_value(value).ok().map(|df| (name, df)) - }) - .map(|(name, df)| { - Value::record( - record! { - "name" => Value::string(name, call.head), - "columns" => Value::int(df.as_ref().width() as i64, call.head), - "rows" => Value::int(df.as_ref().height() as i64, call.head), - }, - call.head, - ) - }) - .collect::>(); - - let list = Value::list(vals, call.head); - - Ok(list.into_pipeline_data()) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/melt.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/melt.rs deleted file mode 100644 index 6379e9270e..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/melt.rs +++ /dev/null @@ -1,248 +0,0 @@ -use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct MeltDF; - -impl Command for MeltDF { - fn name(&self) -> &str { - "dfr melt" - } - - fn usage(&self) -> &str { - "Unpivot a DataFrame from wide to long format." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required_named( - "columns", - SyntaxShape::Table(vec![]), - "column names for melting", - Some('c'), - ) - .required_named( - "values", - SyntaxShape::Table(vec![]), - "column names used as value columns", - Some('v'), - ) - .named( - "variable-name", - SyntaxShape::String, - "optional name for variable column", - Some('r'), - ) - .named( - "value-name", - SyntaxShape::String, - "optional name for value column", - Some('l'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "melt dataframe", - example: - "[[a b c d]; [x 1 4 a] [y 2 5 b] [z 3 6 c]] | dfr into-df | dfr melt -c [b c] -v [a d]", - result: Some( - NuDataFrame::try_from_columns(vec![ - Column::new( - "b".to_string(), - vec![ - Value::test_int(1), - Value::test_int(2), - Value::test_int(3), - Value::test_int(1), - Value::test_int(2), - Value::test_int(3), - ], - ), - Column::new( - "c".to_string(), - vec![ - Value::test_int(4), - Value::test_int(5), - Value::test_int(6), - Value::test_int(4), - Value::test_int(5), - Value::test_int(6), - ], - ), - Column::new( - "variable".to_string(), - vec![ - Value::test_string("a"), - Value::test_string("a"), - Value::test_string("a"), - Value::test_string("d"), - Value::test_string("d"), - Value::test_string("d"), - ], - ), - Column::new( - "value".to_string(), - vec![ - Value::test_string("x"), - Value::test_string("y"), - Value::test_string("z"), - Value::test_string("a"), - Value::test_string("b"), - Value::test_string("c"), - ], - ), - ], None) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let id_col: Vec = call - .get_flag(engine_state, stack, "columns")? - .expect("required value"); - let val_col: Vec = call - .get_flag(engine_state, stack, "values")? - .expect("required value"); - - let value_name: Option> = call.get_flag(engine_state, stack, "value-name")?; - let variable_name: Option> = - call.get_flag(engine_state, stack, "variable-name")?; - - let (id_col_string, id_col_span) = convert_columns_string(id_col, call.head)?; - let (val_col_string, val_col_span) = convert_columns_string(val_col, call.head)?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - check_column_datatypes(df.as_ref(), &id_col_string, id_col_span)?; - check_column_datatypes(df.as_ref(), &val_col_string, val_col_span)?; - - let mut res = df - .as_ref() - .melt(&id_col_string, &val_col_string) - .map_err(|e| ShellError::GenericError { - error: "Error calculating melt".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - if let Some(name) = &variable_name { - res.rename("variable", &name.item) - .map_err(|e| ShellError::GenericError { - error: "Error renaming column".into(), - msg: e.to_string(), - span: Some(name.span), - help: None, - inner: vec![], - })?; - } - - if let Some(name) = &value_name { - res.rename("value", &name.item) - .map_err(|e| ShellError::GenericError { - error: "Error renaming column".into(), - msg: e.to_string(), - span: Some(name.span), - help: None, - inner: vec![], - })?; - } - - Ok(PipelineData::Value( - NuDataFrame::dataframe_into_value(res, call.head), - None, - )) -} - -fn check_column_datatypes>( - df: &polars::prelude::DataFrame, - cols: &[T], - col_span: Span, -) -> Result<(), ShellError> { - if cols.is_empty() { - return Err(ShellError::GenericError { - error: "Merge error".into(), - msg: "empty column list".into(), - span: Some(col_span), - help: None, - inner: vec![], - }); - } - - // Checking if they are same type - if cols.len() > 1 { - for w in cols.windows(2) { - let l_series = df - .column(w[0].as_ref()) - .map_err(|e| ShellError::GenericError { - error: "Error selecting columns".into(), - msg: e.to_string(), - span: Some(col_span), - help: None, - inner: vec![], - })?; - - let r_series = df - .column(w[1].as_ref()) - .map_err(|e| ShellError::GenericError { - error: "Error selecting columns".into(), - msg: e.to_string(), - span: Some(col_span), - help: None, - inner: vec![], - })?; - - if l_series.dtype() != r_series.dtype() { - return Err(ShellError::GenericError { - error: "Merge error".into(), - msg: "found different column types in list".into(), - span: Some(col_span), - help: Some(format!( - "datatypes {} and {} are incompatible", - l_series.dtype(), - r_series.dtype() - )), - inner: vec![], - }); - } - } - } - - Ok(()) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(MeltDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/mod.rs deleted file mode 100644 index db7a5c9312..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/mod.rs +++ /dev/null @@ -1,114 +0,0 @@ -mod append; -mod cast; -mod columns; -mod drop; -mod drop_duplicates; -mod drop_nulls; -mod dtypes; -mod dummies; -mod filter_with; -mod first; -mod get; -mod last; -mod list; -mod melt; -mod open; -mod query_df; -mod rename; -mod sample; -mod schema; -mod shape; -mod slice; -mod sql_context; -mod sql_expr; -mod summary; -mod take; -mod to_arrow; -mod to_avro; -mod to_csv; -mod to_df; -mod to_json_lines; -mod to_nu; -mod to_parquet; -mod with_column; - -use nu_protocol::engine::StateWorkingSet; - -pub use self::open::OpenDataFrame; -pub use append::AppendDF; -pub use cast::CastDF; -pub use columns::ColumnsDF; -pub use drop::DropDF; -pub use drop_duplicates::DropDuplicates; -pub use drop_nulls::DropNulls; -pub use dtypes::DataTypes; -pub use dummies::Dummies; -pub use filter_with::FilterWith; -pub use first::FirstDF; -pub use get::GetDF; -pub use last::LastDF; -pub use list::ListDF; -pub use melt::MeltDF; -pub use query_df::QueryDf; -pub use rename::RenameDF; -pub use sample::SampleDF; -pub use schema::SchemaDF; -pub use shape::ShapeDF; -pub use slice::SliceDF; -pub use sql_context::SQLContext; -pub use summary::Summary; -pub use take::TakeDF; -pub use to_arrow::ToArrow; -pub use to_avro::ToAvro; -pub use to_csv::ToCSV; -pub use to_df::ToDataFrame; -pub use to_json_lines::ToJsonLines; -pub use to_nu::ToNu; -pub use to_parquet::ToParquet; -pub use with_column::WithColumn; - -pub fn add_eager_decls(working_set: &mut StateWorkingSet) { - macro_rules! bind_command { - ( $command:expr ) => { - working_set.add_decl(Box::new($command)); - }; - ( $( $command:expr ),* ) => { - $( working_set.add_decl(Box::new($command)); )* - }; - } - - // Dataframe commands - bind_command!( - AppendDF, - CastDF, - ColumnsDF, - DataTypes, - Summary, - DropDF, - DropDuplicates, - DropNulls, - Dummies, - FilterWith, - FirstDF, - GetDF, - LastDF, - ListDF, - MeltDF, - OpenDataFrame, - QueryDf, - RenameDF, - SampleDF, - SchemaDF, - ShapeDF, - SliceDF, - TakeDF, - ToArrow, - ToAvro, - ToCSV, - ToDataFrame, - ToNu, - ToParquet, - ToJsonLines, - WithColumn - ); -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/open.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/open.rs deleted file mode 100644 index 38d0d0c49f..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/open.rs +++ /dev/null @@ -1,518 +0,0 @@ -use crate::dataframe::values::{NuDataFrame, NuLazyFrame, NuSchema}; -use nu_engine::command_prelude::*; - -use polars::prelude::{ - CsvEncoding, CsvReader, IpcReader, JsonFormat, JsonReader, LazyCsvReader, LazyFileListReader, - LazyFrame, ParallelStrategy, ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader, -}; -use polars_io::{avro::AvroReader, HiveOptions}; -use std::{fs::File, io::BufReader, path::PathBuf}; - -#[derive(Clone)] -pub struct OpenDataFrame; - -impl Command for OpenDataFrame { - fn name(&self) -> &str { - "dfr open" - } - - fn usage(&self) -> &str { - "Opens CSV, JSON, JSON lines, arrow, avro, or parquet file to create dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "file", - SyntaxShape::Filepath, - "file path to load values from", - ) - .switch("lazy", "creates a lazy dataframe", Some('l')) - .named( - "type", - SyntaxShape::String, - "File type: csv, tsv, json, parquet, arrow, avro. If omitted, derive from file extension", - Some('t'), - ) - .named( - "delimiter", - SyntaxShape::String, - "file delimiter character. CSV file", - Some('d'), - ) - .switch( - "no-header", - "Indicates if file doesn't have header. CSV file", - None, - ) - .named( - "infer-schema", - SyntaxShape::Number, - "Number of rows to infer the schema of the file. CSV file", - None, - ) - .named( - "skip-rows", - SyntaxShape::Number, - "Number of rows to skip from file. CSV file", - None, - ) - .named( - "columns", - SyntaxShape::List(Box::new(SyntaxShape::String)), - "Columns to be selected from csv file. CSV and Parquet file", - None, - ) - .named( - "schema", - SyntaxShape::Record(vec![]), - r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#, - Some('s') - ) - .input_output_type(Type::Any, Type::Custom("dataframe".into())) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Takes a file name and creates a dataframe", - example: "dfr open test.csv", - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - command(engine_state, stack, call) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let file: Spanned = call.req(engine_state, stack, 0)?; - - let type_option: Option> = call.get_flag(engine_state, stack, "type")?; - - let type_id = match &type_option { - Some(ref t) => Some((t.item.to_owned(), "Invalid type", t.span)), - None => file.item.extension().map(|e| { - ( - e.to_string_lossy().into_owned(), - "Invalid extension", - file.span, - ) - }), - }; - - match type_id { - Some((e, msg, blamed)) => match e.as_str() { - "csv" | "tsv" => from_csv(engine_state, stack, call), - "parquet" | "parq" => from_parquet(engine_state, stack, call), - "ipc" | "arrow" => from_ipc(engine_state, stack, call), - "json" => from_json(engine_state, stack, call), - "jsonl" => from_jsonl(engine_state, stack, call), - "avro" => from_avro(engine_state, stack, call), - _ => Err(ShellError::FileNotFoundCustom { - msg: format!( - "{msg}. Supported values: csv, tsv, parquet, ipc, arrow, json, jsonl, avro" - ), - span: blamed, - }), - }, - None => Err(ShellError::FileNotFoundCustom { - msg: "File without extension".into(), - span: file.span, - }), - } - .map(|value| PipelineData::Value(value, None)) -} - -fn from_parquet( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - if call.has_flag(engine_state, stack, "lazy")? { - let file: String = call.req(engine_state, stack, 0)?; - let args = ScanArgsParquet { - n_rows: None, - cache: true, - parallel: ParallelStrategy::Auto, - rechunk: false, - row_index: None, - low_memory: false, - cloud_options: None, - use_statistics: false, - hive_options: HiveOptions::default(), - }; - - let df: NuLazyFrame = LazyFrame::scan_parquet(file, args) - .map_err(|e| ShellError::GenericError { - error: "Parquet reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - df.into_value(call.head) - } else { - let file: Spanned = call.req(engine_state, stack, 0)?; - let columns: Option> = call.get_flag(engine_state, stack, "columns")?; - - let r = File::open(&file.item).map_err(|e| ShellError::GenericError { - error: "Error opening file".into(), - msg: e.to_string(), - span: Some(file.span), - help: None, - inner: vec![], - })?; - let reader = ParquetReader::new(r); - - let reader = match columns { - None => reader, - Some(columns) => reader.with_columns(Some(columns)), - }; - - let df: NuDataFrame = reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "Parquet reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(df.into_value(call.head)) - } -} - -fn from_avro( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let file: Spanned = call.req(engine_state, stack, 0)?; - let columns: Option> = call.get_flag(engine_state, stack, "columns")?; - - let r = File::open(&file.item).map_err(|e| ShellError::GenericError { - error: "Error opening file".into(), - msg: e.to_string(), - span: Some(file.span), - help: None, - inner: vec![], - })?; - let reader = AvroReader::new(r); - - let reader = match columns { - None => reader, - Some(columns) => reader.with_columns(Some(columns)), - }; - - let df: NuDataFrame = reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "Avro reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(df.into_value(call.head)) -} - -fn from_ipc( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - if call.has_flag(engine_state, stack, "lazy")? { - let file: String = call.req(engine_state, stack, 0)?; - let args = ScanArgsIpc { - n_rows: None, - cache: true, - rechunk: false, - row_index: None, - memory_map: true, - cloud_options: None, - }; - - let df: NuLazyFrame = LazyFrame::scan_ipc(file, args) - .map_err(|e| ShellError::GenericError { - error: "IPC reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - df.into_value(call.head) - } else { - let file: Spanned = call.req(engine_state, stack, 0)?; - let columns: Option> = call.get_flag(engine_state, stack, "columns")?; - - let r = File::open(&file.item).map_err(|e| ShellError::GenericError { - error: "Error opening file".into(), - msg: e.to_string(), - span: Some(file.span), - help: None, - inner: vec![], - })?; - let reader = IpcReader::new(r); - - let reader = match columns { - None => reader, - Some(columns) => reader.with_columns(Some(columns)), - }; - - let df: NuDataFrame = reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "IPC reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(df.into_value(call.head)) - } -} - -fn from_json( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let file: Spanned = call.req(engine_state, stack, 0)?; - let file = File::open(&file.item).map_err(|e| ShellError::GenericError { - error: "Error opening file".into(), - msg: e.to_string(), - span: Some(file.span), - help: None, - inner: vec![], - })?; - let maybe_schema = call - .get_flag(engine_state, stack, "schema")? - .map(|schema| NuSchema::try_from(&schema)) - .transpose()?; - - let buf_reader = BufReader::new(file); - let reader = JsonReader::new(buf_reader); - - let reader = match maybe_schema { - Some(schema) => reader.with_schema(schema.into()), - None => reader, - }; - - let df: NuDataFrame = reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "Json reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(df.into_value(call.head)) -} - -fn from_jsonl( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let infer_schema: Option = call.get_flag(engine_state, stack, "infer-schema")?; - let maybe_schema = call - .get_flag(engine_state, stack, "schema")? - .map(|schema| NuSchema::try_from(&schema)) - .transpose()?; - let file: Spanned = call.req(engine_state, stack, 0)?; - let file = File::open(&file.item).map_err(|e| ShellError::GenericError { - error: "Error opening file".into(), - msg: e.to_string(), - span: Some(file.span), - help: None, - inner: vec![], - })?; - - let buf_reader = BufReader::new(file); - let reader = JsonReader::new(buf_reader) - .with_json_format(JsonFormat::JsonLines) - .infer_schema_len(infer_schema); - - let reader = match maybe_schema { - Some(schema) => reader.with_schema(schema.into()), - None => reader, - }; - - let df: NuDataFrame = reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "Json lines reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(df.into_value(call.head)) -} - -fn from_csv( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let delimiter: Option> = call.get_flag(engine_state, stack, "delimiter")?; - let no_header: bool = call.has_flag(engine_state, stack, "no-header")?; - let infer_schema: Option = call.get_flag(engine_state, stack, "infer-schema")?; - let skip_rows: Option = call.get_flag(engine_state, stack, "skip-rows")?; - let columns: Option> = call.get_flag(engine_state, stack, "columns")?; - - let maybe_schema = call - .get_flag(engine_state, stack, "schema")? - .map(|schema| NuSchema::try_from(&schema)) - .transpose()?; - - if call.has_flag(engine_state, stack, "lazy")? { - let file: String = call.req(engine_state, stack, 0)?; - let csv_reader = LazyCsvReader::new(file); - - let csv_reader = match delimiter { - None => csv_reader, - Some(d) => { - if d.item.len() != 1 { - return Err(ShellError::GenericError { - error: "Incorrect delimiter".into(), - msg: "Delimiter has to be one character".into(), - span: Some(d.span), - help: None, - inner: vec![], - }); - } else { - let delimiter = match d.item.chars().next() { - Some(d) => d as u8, - None => unreachable!(), - }; - csv_reader.with_separator(delimiter) - } - } - }; - - let csv_reader = csv_reader.has_header(!no_header); - - let csv_reader = match maybe_schema { - Some(schema) => csv_reader.with_schema(Some(schema.into())), - None => csv_reader, - }; - - let csv_reader = match infer_schema { - None => csv_reader, - Some(r) => csv_reader.with_infer_schema_length(Some(r)), - }; - - let csv_reader = match skip_rows { - None => csv_reader, - Some(r) => csv_reader.with_skip_rows(r), - }; - - let df: NuLazyFrame = csv_reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "Parquet reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - df.into_value(call.head) - } else { - let file: Spanned = call.req(engine_state, stack, 0)?; - let csv_reader = CsvReader::from_path(&file.item) - .map_err(|e| ShellError::GenericError { - error: "Error creating CSV reader".into(), - msg: e.to_string(), - span: Some(file.span), - help: None, - inner: vec![], - })? - .with_encoding(CsvEncoding::LossyUtf8); - - let csv_reader = match delimiter { - None => csv_reader, - Some(d) => { - if d.item.len() != 1 { - return Err(ShellError::GenericError { - error: "Incorrect delimiter".into(), - msg: "Delimiter has to be one character".into(), - span: Some(d.span), - help: None, - inner: vec![], - }); - } else { - let delimiter = match d.item.chars().next() { - Some(d) => d as u8, - None => unreachable!(), - }; - csv_reader.with_separator(delimiter) - } - } - }; - - let csv_reader = csv_reader.has_header(!no_header); - - let csv_reader = match maybe_schema { - Some(schema) => csv_reader.with_schema(Some(schema.into())), - None => csv_reader, - }; - - let csv_reader = match infer_schema { - None => csv_reader, - Some(r) => csv_reader.infer_schema(Some(r)), - }; - - let csv_reader = match skip_rows { - None => csv_reader, - Some(r) => csv_reader.with_skip_rows(r), - }; - - let csv_reader = match columns { - None => csv_reader, - Some(columns) => csv_reader.with_columns(Some(columns)), - }; - - let df: NuDataFrame = csv_reader - .finish() - .map_err(|e| ShellError::GenericError { - error: "Parquet reader error".into(), - msg: format!("{e:?}"), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(df.into_value(call.head)) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/query_df.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/query_df.rs deleted file mode 100644 index 4088e00afa..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/query_df.rs +++ /dev/null @@ -1,104 +0,0 @@ -use crate::dataframe::{ - eager::SQLContext, - values::{Column, NuDataFrame, NuLazyFrame}, -}; -use nu_engine::command_prelude::*; - -// attribution: -// sql_context.rs, and sql_expr.rs were copied from polars-sql. thank you. -// maybe we should just use the crate at some point but it's not published yet. -// https://github.com/pola-rs/polars/tree/master/polars-sql - -#[derive(Clone)] -pub struct QueryDf; - -impl Command for QueryDf { - fn name(&self) -> &str { - "dfr query" - } - - fn usage(&self) -> &str { - "Query dataframe using SQL. Note: The dataframe is always named 'df' in your query's from clause." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("sql", SyntaxShape::String, "sql query") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn search_terms(&self) -> Vec<&str> { - vec!["dataframe", "sql", "search"] - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Query dataframe using SQL", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr query 'select a from df'", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let sql_query: String = call.req(engine_state, stack, 0)?; - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut ctx = SQLContext::new(); - ctx.register("df", &df.df); - let df_sql = ctx - .execute(&sql_query) - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - let lazy = NuLazyFrame::new(false, df_sql); - - let eager = lazy.collect(call.head)?; - let value = Value::custom(Box::new(eager), call.head); - - Ok(PipelineData::Value(value, None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(QueryDf {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs deleted file mode 100644 index 0cb75f34f2..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs +++ /dev/null @@ -1,185 +0,0 @@ -use crate::dataframe::{ - utils::extract_strings, - values::{Column, NuDataFrame, NuLazyFrame}, -}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct RenameDF; - -impl Command for RenameDF { - fn name(&self) -> &str { - "dfr rename" - } - - fn usage(&self) -> &str { - "Rename a dataframe column." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "columns", - SyntaxShape::Any, - "Column(s) to be renamed. A string or list of strings", - ) - .required( - "new names", - SyntaxShape::Any, - "New names for the selected column(s). A string or list of strings", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe or lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Renames a series", - example: "[5 6 7 8] | dfr into-df | dfr rename '0' new_name", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "new_name".to_string(), - vec![ - Value::test_int(5), - Value::test_int(6), - Value::test_int(7), - Value::test_int(8), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Renames a dataframe column", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename a a_new", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a_new".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Renames two dataframe columns", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename [a b] [a_new b_new]", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a_new".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b_new".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuLazyFrame::can_downcast(&value) { - let df = NuLazyFrame::try_from_value(value)?; - command_lazy(engine_state, stack, call, df) - } else { - let df = NuDataFrame::try_from_value(value)?; - command_eager(engine_state, stack, call, df) - } - } -} - -fn command_eager( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - mut df: NuDataFrame, -) -> Result { - let columns: Value = call.req(engine_state, stack, 0)?; - let columns = extract_strings(columns)?; - - let new_names: Value = call.req(engine_state, stack, 1)?; - let new_names = extract_strings(new_names)?; - - for (from, to) in columns.iter().zip(new_names.iter()) { - df.as_mut() - .rename(from, to) - .map_err(|e| ShellError::GenericError { - error: "Error renaming".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - } - - Ok(PipelineData::Value(df.into_value(call.head), None)) -} - -fn command_lazy( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - lazy: NuLazyFrame, -) -> Result { - let columns: Value = call.req(engine_state, stack, 0)?; - let columns = extract_strings(columns)?; - - let new_names: Value = call.req(engine_state, stack, 1)?; - let new_names = extract_strings(new_names)?; - - if columns.len() != new_names.len() { - let value: Value = call.req(engine_state, stack, 1)?; - return Err(ShellError::IncompatibleParametersSingle { - msg: "New name list has different size to column list".into(), - span: value.span(), - }); - } - - let lazy = lazy.into_polars(); - let lazy: NuLazyFrame = lazy.rename(&columns, &new_names).into(); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(RenameDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/sample.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/sample.rs deleted file mode 100644 index 2387cca489..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/sample.rs +++ /dev/null @@ -1,127 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::{prelude::NamedFrom, series::Series}; - -#[derive(Clone)] -pub struct SampleDF; - -impl Command for SampleDF { - fn name(&self) -> &str { - "dfr sample" - } - - fn usage(&self) -> &str { - "Create sample dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .named( - "n-rows", - SyntaxShape::Int, - "number of rows to be taken from dataframe", - Some('n'), - ) - .named( - "fraction", - SyntaxShape::Number, - "fraction of dataframe to be taken", - Some('f'), - ) - .named( - "seed", - SyntaxShape::Number, - "seed for the selection", - Some('s'), - ) - .switch("replace", "sample with replace", Some('e')) - .switch("shuffle", "shuffle sample", Some('u')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Sample rows from dataframe", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr sample --n-rows 1", - result: None, // No expected value because sampling is random - }, - Example { - description: "Shows sample row using fraction and replace", - example: - "[[a b]; [1 2] [3 4] [5 6]] | dfr into-df | dfr sample --fraction 0.5 --replace", - result: None, // No expected value because sampling is random - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let rows: Option> = call.get_flag(engine_state, stack, "n-rows")?; - let fraction: Option> = call.get_flag(engine_state, stack, "fraction")?; - let seed: Option = call - .get_flag::(engine_state, stack, "seed")? - .map(|val| val as u64); - let replace: bool = call.has_flag(engine_state, stack, "replace")?; - let shuffle: bool = call.has_flag(engine_state, stack, "shuffle")?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - match (rows, fraction) { - (Some(rows), None) => df - .as_ref() - .sample_n(&Series::new("s", &[rows.item]), replace, shuffle, seed) - .map_err(|e| ShellError::GenericError { - error: "Error creating sample".into(), - msg: e.to_string(), - span: Some(rows.span), - help: None, - inner: vec![], - }), - (None, Some(frac)) => df - .as_ref() - .sample_frac(&Series::new("frac", &[frac.item]), replace, shuffle, seed) - .map_err(|e| ShellError::GenericError { - error: "Error creating sample".into(), - msg: e.to_string(), - span: Some(frac.span), - help: None, - inner: vec![], - }), - (Some(_), Some(_)) => Err(ShellError::GenericError { - error: "Incompatible flags".into(), - msg: "Only one selection criterion allowed".into(), - span: Some(call.head), - help: None, - inner: vec![], - }), - (None, None) => Err(ShellError::GenericError { - error: "No selection".into(), - msg: "No selection criterion was found".into(), - span: Some(call.head), - help: Some("Perhaps you want to use the flag -n or -f".into()), - inner: vec![], - }), - } - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/schema.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/schema.rs deleted file mode 100644 index cf887482bd..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/schema.rs +++ /dev/null @@ -1,112 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct SchemaDF; - -impl Command for SchemaDF { - fn name(&self) -> &str { - "dfr schema" - } - - fn usage(&self) -> &str { - "Show schema for a dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .switch("datatype-list", "creates a lazy dataframe", Some('l')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Dataframe schema", - example: r#"[[a b]; [1 "foo"] [3 "bar"]] | dfr into-df | dfr schema"#, - result: Some(Value::record( - record! { - "a" => Value::string("i64", Span::test_data()), - "b" => Value::string("str", Span::test_data()), - }, - Span::test_data(), - )), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - if call.has_flag(engine_state, stack, "datatype-list")? { - Ok(PipelineData::Value(datatype_list(Span::unknown()), None)) - } else { - command(engine_state, stack, call, input) - } - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let schema = df.schema(); - let value: Value = schema.into(); - Ok(PipelineData::Value(value, None)) -} - -fn datatype_list(span: Span) -> Value { - let types: Vec = [ - ("null", ""), - ("bool", ""), - ("u8", ""), - ("u16", ""), - ("u32", ""), - ("u64", ""), - ("i8", ""), - ("i16", ""), - ("i32", ""), - ("i64", ""), - ("f32", ""), - ("f64", ""), - ("str", ""), - ("binary", ""), - ("date", ""), - ("datetime", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns. Timezone wildcard is *. Other Timezone examples: UTC, America/Los_Angeles."), - ("duration", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns."), - ("time", ""), - ("object", ""), - ("unknown", ""), - ("list", ""), - ] - .iter() - .map(|(dtype, note)| { - Value::record(record! { - "dtype" => Value::string(*dtype, span), - "note" => Value::string(*note, span), - }, - span) - }) - .collect(); - Value::list(types, span) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(SchemaDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/shape.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/shape.rs deleted file mode 100644 index 6e5e7fa9d3..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/shape.rs +++ /dev/null @@ -1,82 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ShapeDF; - -impl Command for ShapeDF { - fn name(&self) -> &str { - "dfr shape" - } - - fn usage(&self) -> &str { - "Shows column and row size for a dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Shows row and column shape", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr shape", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("rows".to_string(), vec![Value::test_int(2)]), - Column::new("columns".to_string(), vec![Value::test_int(2)]), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let rows = Value::int(df.as_ref().height() as i64, call.head); - - let cols = Value::int(df.as_ref().width() as i64, call.head); - - let rows_col = Column::new("rows".to_string(), vec![rows]); - let cols_col = Column::new("columns".to_string(), vec![cols]); - - NuDataFrame::try_from_columns(vec![rows_col, cols_col], None) - .map(|df| PipelineData::Value(df.into_value(call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ShapeDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/slice.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/slice.rs deleted file mode 100644 index 48906cba2c..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/slice.rs +++ /dev/null @@ -1,84 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct SliceDF; - -impl Command for SliceDF { - fn name(&self) -> &str { - "dfr slice" - } - - fn usage(&self) -> &str { - "Creates new dataframe from a slice of rows." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("offset", SyntaxShape::Int, "start of slice") - .required("size", SyntaxShape::Int, "size of slice") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Create new dataframe from a slice of the rows", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr slice 0 1", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_int(1)]), - Column::new("b".to_string(), vec![Value::test_int(2)]), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let offset: i64 = call.req(engine_state, stack, 0)?; - let size: usize = call.req(engine_state, stack, 1)?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let res = df.as_ref().slice(offset, size); - - Ok(PipelineData::Value( - NuDataFrame::dataframe_into_value(res, call.head), - None, - )) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(SliceDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs deleted file mode 100644 index f558904344..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs +++ /dev/null @@ -1,228 +0,0 @@ -use crate::dataframe::eager::sql_expr::parse_sql_expr; -use polars::error::{ErrString, PolarsError}; -use polars::prelude::{col, DataFrame, DataType, IntoLazy, LazyFrame}; -use sqlparser::ast::{ - Expr as SqlExpr, GroupByExpr, Select, SelectItem, SetExpr, Statement, TableFactor, - Value as SQLValue, -}; -use sqlparser::dialect::GenericDialect; -use sqlparser::parser::Parser; -use std::collections::HashMap; - -#[derive(Default)] -pub struct SQLContext { - table_map: HashMap, - dialect: GenericDialect, -} - -impl SQLContext { - pub fn new() -> Self { - Self { - table_map: HashMap::new(), - dialect: GenericDialect, - } - } - - pub fn register(&mut self, name: &str, df: &DataFrame) { - self.table_map.insert(name.to_owned(), df.clone().lazy()); - } - - fn execute_select(&self, select_stmt: &Select) -> Result { - // Determine involved dataframe - // Implicit join require some more work in query parsers, Explicit join are preferred for now. - let tbl = select_stmt.from.first().ok_or_else(|| { - PolarsError::ComputeError(ErrString::from("No table found in select statement")) - })?; - let mut alias_map = HashMap::new(); - let tbl_name = match &tbl.relation { - TableFactor::Table { name, alias, .. } => { - let tbl_name = name - .0 - .first() - .ok_or_else(|| { - PolarsError::ComputeError(ErrString::from( - "No table found in select statement", - )) - })? - .value - .to_string(); - if self.table_map.contains_key(&tbl_name) { - if let Some(alias) = alias { - alias_map.insert(alias.name.value.clone(), tbl_name.to_owned()); - }; - tbl_name - } else { - return Err(PolarsError::ComputeError( - format!("Table name {tbl_name} was not found").into(), - )); - } - } - // Support bare table, optional with alias for now - _ => return Err(PolarsError::ComputeError("Not implemented".into())), - }; - let df = &self.table_map[&tbl_name]; - let mut raw_projection_before_alias: HashMap = HashMap::new(); - let mut contain_wildcard = false; - // Filter Expression - let df = match select_stmt.selection.as_ref() { - Some(expr) => { - let filter_expression = parse_sql_expr(expr)?; - df.clone().filter(filter_expression) - } - None => df.clone(), - }; - // Column Projections - let projection = select_stmt - .projection - .iter() - .enumerate() - .map(|(i, select_item)| { - Ok(match select_item { - SelectItem::UnnamedExpr(expr) => { - let expr = parse_sql_expr(expr)?; - raw_projection_before_alias.insert(format!("{expr:?}"), i); - expr - } - SelectItem::ExprWithAlias { expr, alias } => { - let expr = parse_sql_expr(expr)?; - raw_projection_before_alias.insert(format!("{expr:?}"), i); - expr.alias(&alias.value) - } - SelectItem::QualifiedWildcard(_, _) | SelectItem::Wildcard(_) => { - contain_wildcard = true; - col("*") - } - }) - }) - .collect::, PolarsError>>()?; - // Check for group by - // After projection since there might be number. - let group_by = match &select_stmt.group_by { - GroupByExpr::All => - Err( - PolarsError::ComputeError("Group-By Error: Only positive number or expression are supported, not all".into()) - )?, - GroupByExpr::Expressions(expressions) => expressions - } - .iter() - .map( - |e|match e { - SqlExpr::Value(SQLValue::Number(idx, _)) => { - let idx = match idx.parse::() { - Ok(0)| Err(_) => Err( - PolarsError::ComputeError( - format!("Group-By Error: Only positive number or expression are supported, got {idx}").into() - )), - Ok(idx) => Ok(idx) - }?; - Ok(projection[idx].clone()) - } - SqlExpr::Value(_) => Err( - PolarsError::ComputeError("Group-By Error: Only positive number or expression are supported".into()) - ), - _ => parse_sql_expr(e) - } - ) - .collect::, PolarsError>>()?; - - let df = if group_by.is_empty() { - df.select(projection) - } else { - // check groupby and projection due to difference between SQL and polars - // Return error on wild card, shouldn't process this - if contain_wildcard { - return Err(PolarsError::ComputeError( - "Group-By Error: Can't process wildcard in group-by".into(), - )); - } - // Default polars group by will have group by columns at the front - // need some container to contain position of group by columns and its position - // at the final agg projection, check the schema for the existence of group by column - // and its projections columns, keeping the original index - let (exclude_expr, groupby_pos): (Vec<_>, Vec<_>) = group_by - .iter() - .map(|expr| raw_projection_before_alias.get(&format!("{expr:?}"))) - .enumerate() - .filter(|(_, proj_p)| proj_p.is_some()) - .map(|(gb_p, proj_p)| (*proj_p.unwrap_or(&0), (*proj_p.unwrap_or(&0), gb_p))) - .unzip(); - let (agg_projection, agg_proj_pos): (Vec<_>, Vec<_>) = projection - .iter() - .enumerate() - .filter(|(i, _)| !exclude_expr.contains(i)) - .enumerate() - .map(|(agg_pj, (proj_p, expr))| (expr.clone(), (proj_p, agg_pj + group_by.len()))) - .unzip(); - let agg_df = df.group_by(group_by).agg(agg_projection); - let mut final_proj_pos = groupby_pos - .into_iter() - .chain(agg_proj_pos) - .collect::>(); - - final_proj_pos.sort_by(|(proj_pa, _), (proj_pb, _)| proj_pa.cmp(proj_pb)); - let final_proj = final_proj_pos - .into_iter() - .map(|(_, shm_p)| { - col(agg_df - .clone() - // FIXME: had to do this mess to get get_index to work, not sure why. need help - .collect() - .unwrap_or_default() - .schema() - .get_at_index(shm_p) - .unwrap_or((&"".into(), &DataType::Null)) - .0) - }) - .collect::>(); - agg_df.select(final_proj) - }; - Ok(df) - } - - pub fn execute(&self, query: &str) -> Result { - let ast = Parser::parse_sql(&self.dialect, query) - .map_err(|e| PolarsError::ComputeError(format!("{e:?}").into()))?; - if ast.len() != 1 { - Err(PolarsError::ComputeError( - "One and only one statement at a time please".into(), - )) - } else { - let ast = ast - .first() - .ok_or_else(|| PolarsError::ComputeError(ErrString::from("No statement found")))?; - Ok(match ast { - Statement::Query(query) => { - let rs = match &*query.body { - SetExpr::Select(select_stmt) => self.execute_select(select_stmt)?, - _ => { - return Err(PolarsError::ComputeError( - "INSERT, UPDATE is not supported for polars".into(), - )) - } - }; - match &query.limit { - Some(SqlExpr::Value(SQLValue::Number(nrow, _))) => { - let nrow = nrow.parse().map_err(|err| { - PolarsError::ComputeError( - format!("Conversion Error: {err:?}").into(), - ) - })?; - rs.limit(nrow) - } - None => rs, - _ => { - return Err(PolarsError::ComputeError( - "Only support number argument to LIMIT clause".into(), - )) - } - } - } - _ => { - return Err(PolarsError::ComputeError( - format!("Statement type {ast:?} is not supported").into(), - )) - } - }) - } - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/sql_expr.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/sql_expr.rs deleted file mode 100644 index 9c0728ea5f..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/sql_expr.rs +++ /dev/null @@ -1,200 +0,0 @@ -use polars::error::PolarsError; -use polars::prelude::{col, lit, DataType, Expr, LiteralValue, PolarsResult as Result, TimeUnit}; - -use sqlparser::ast::{ - ArrayElemTypeDef, BinaryOperator as SQLBinaryOperator, DataType as SQLDataType, - Expr as SqlExpr, Function as SQLFunction, Value as SqlValue, WindowType, -}; - -fn map_sql_polars_datatype(data_type: &SQLDataType) -> Result { - Ok(match data_type { - SQLDataType::Char(_) - | SQLDataType::Varchar(_) - | SQLDataType::Uuid - | SQLDataType::Clob(_) - | SQLDataType::Text - | SQLDataType::String(_) => DataType::String, - SQLDataType::Float(_) => DataType::Float32, - SQLDataType::Real => DataType::Float32, - SQLDataType::Double => DataType::Float64, - SQLDataType::TinyInt(_) => DataType::Int8, - SQLDataType::UnsignedTinyInt(_) => DataType::UInt8, - SQLDataType::SmallInt(_) => DataType::Int16, - SQLDataType::UnsignedSmallInt(_) => DataType::UInt16, - SQLDataType::Int(_) => DataType::Int32, - SQLDataType::UnsignedInt(_) => DataType::UInt32, - SQLDataType::BigInt(_) => DataType::Int64, - SQLDataType::UnsignedBigInt(_) => DataType::UInt64, - - SQLDataType::Boolean => DataType::Boolean, - SQLDataType::Date => DataType::Date, - SQLDataType::Time(_, _) => DataType::Time, - SQLDataType::Timestamp(_, _) => DataType::Datetime(TimeUnit::Microseconds, None), - SQLDataType::Interval => DataType::Duration(TimeUnit::Microseconds), - SQLDataType::Array(array_type_def) => match array_type_def { - ArrayElemTypeDef::AngleBracket(inner_type) - | ArrayElemTypeDef::SquareBracket(inner_type) => { - DataType::List(Box::new(map_sql_polars_datatype(inner_type)?)) - } - _ => { - return Err(PolarsError::ComputeError( - "SQL Datatype Array(None) was not supported in polars-sql yet!".into(), - )) - } - }, - _ => { - return Err(PolarsError::ComputeError( - format!("SQL Datatype {data_type:?} was not supported in polars-sql yet!").into(), - )) - } - }) -} - -fn cast_(expr: Expr, data_type: &SQLDataType) -> Result { - let polars_type = map_sql_polars_datatype(data_type)?; - Ok(expr.cast(polars_type)) -} - -fn binary_op_(left: Expr, right: Expr, op: &SQLBinaryOperator) -> Result { - Ok(match op { - SQLBinaryOperator::Plus => left + right, - SQLBinaryOperator::Minus => left - right, - SQLBinaryOperator::Multiply => left * right, - SQLBinaryOperator::Divide => left / right, - SQLBinaryOperator::Modulo => left % right, - SQLBinaryOperator::StringConcat => { - left.cast(DataType::String) + right.cast(DataType::String) - } - SQLBinaryOperator::Gt => left.gt(right), - SQLBinaryOperator::Lt => left.lt(right), - SQLBinaryOperator::GtEq => left.gt_eq(right), - SQLBinaryOperator::LtEq => left.lt_eq(right), - SQLBinaryOperator::Eq => left.eq(right), - SQLBinaryOperator::NotEq => left.eq(right).not(), - SQLBinaryOperator::And => left.and(right), - SQLBinaryOperator::Or => left.or(right), - SQLBinaryOperator::Xor => left.xor(right), - _ => { - return Err(PolarsError::ComputeError( - format!("SQL Operator {op:?} was not supported in polars-sql yet!").into(), - )) - } - }) -} - -fn literal_expr(value: &SqlValue) -> Result { - Ok(match value { - SqlValue::Number(s, _) => { - // Check for existence of decimal separator dot - if s.contains('.') { - s.parse::().map(lit).map_err(|_| { - PolarsError::ComputeError(format!("Can't parse literal {s:?}").into()) - }) - } else { - s.parse::().map(lit).map_err(|_| { - PolarsError::ComputeError(format!("Can't parse literal {s:?}").into()) - }) - }? - } - SqlValue::SingleQuotedString(s) => lit(s.clone()), - SqlValue::NationalStringLiteral(s) => lit(s.clone()), - SqlValue::HexStringLiteral(s) => lit(s.clone()), - SqlValue::DoubleQuotedString(s) => lit(s.clone()), - SqlValue::Boolean(b) => lit(*b), - SqlValue::Null => Expr::Literal(LiteralValue::Null), - _ => { - return Err(PolarsError::ComputeError( - format!("Parsing SQL Value {value:?} was not supported in polars-sql yet!").into(), - )) - } - }) -} - -pub fn parse_sql_expr(expr: &SqlExpr) -> Result { - Ok(match expr { - SqlExpr::Identifier(e) => col(&e.value), - SqlExpr::BinaryOp { left, op, right } => { - let left = parse_sql_expr(left)?; - let right = parse_sql_expr(right)?; - binary_op_(left, right, op)? - } - SqlExpr::Function(sql_function) => parse_sql_function(sql_function)?, - SqlExpr::Cast { - expr, - data_type, - format: _, - } => cast_(parse_sql_expr(expr)?, data_type)?, - SqlExpr::Nested(expr) => parse_sql_expr(expr)?, - SqlExpr::Value(value) => literal_expr(value)?, - _ => { - return Err(PolarsError::ComputeError( - format!("Expression: {expr:?} was not supported in polars-sql yet!").into(), - )) - } - }) -} - -fn apply_window_spec(expr: Expr, window_type: Option<&WindowType>) -> Result { - Ok(match &window_type { - Some(wtype) => match wtype { - WindowType::WindowSpec(window_spec) => { - // Process for simple window specification, partition by first - let partition_by = window_spec - .partition_by - .iter() - .map(parse_sql_expr) - .collect::>>()?; - expr.over(partition_by) - // Order by and Row range may not be supported at the moment - } - // TODO: make NamedWindow work - WindowType::NamedWindow(_named) => { - return Err(PolarsError::ComputeError( - format!("Expression: {expr:?} was not supported in polars-sql yet!").into(), - )) - } - }, - None => expr, - }) -} - -fn parse_sql_function(sql_function: &SQLFunction) -> Result { - use sqlparser::ast::{FunctionArg, FunctionArgExpr}; - // Function name mostly do not have name space, so it mostly take the first args - let function_name = sql_function.name.0[0].value.to_ascii_lowercase(); - let args = sql_function - .args - .iter() - .map(|arg| match arg { - FunctionArg::Named { arg, .. } => arg, - FunctionArg::Unnamed(arg) => arg, - }) - .collect::>(); - Ok( - match ( - function_name.as_str(), - args.as_slice(), - sql_function.distinct, - ) { - ("sum", [FunctionArgExpr::Expr(expr)], false) => { - apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.sum() - } - ("count", [FunctionArgExpr::Expr(expr)], false) => { - apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.count() - } - ("count", [FunctionArgExpr::Expr(expr)], true) => { - apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.n_unique() - } - // Special case for wildcard args to count function. - ("count", [FunctionArgExpr::Wildcard], false) => lit(1i32).count(), - _ => { - return Err(PolarsError::ComputeError( - format!( - "Function {function_name:?} with args {args:?} was not supported in polars-sql yet!" - ) - .into(), - )) - } - }, - ) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/summary.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/summary.rs deleted file mode 100644 index 845929a52d..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/summary.rs +++ /dev/null @@ -1,279 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::{ - chunked_array::ChunkedArray, - prelude::{ - AnyValue, DataFrame, DataType, Float64Type, IntoSeries, NewChunkedArray, - QuantileInterpolOptions, Series, StringType, - }, -}; - -#[derive(Clone)] -pub struct Summary; - -impl Command for Summary { - fn name(&self) -> &str { - "dfr summary" - } - - fn usage(&self) -> &str { - "For a dataframe, produces descriptive statistics (summary statistics) for its numeric columns." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .category(Category::Custom("dataframe".into())) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .named( - "quantiles", - SyntaxShape::Table(vec![]), - "provide optional quantiles", - Some('q'), - ) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "list dataframe descriptives", - example: "[[a b]; [1 1] [1 1]] | dfr into-df | dfr summary", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "descriptor".to_string(), - vec![ - Value::test_string("count"), - Value::test_string("sum"), - Value::test_string("mean"), - Value::test_string("median"), - Value::test_string("std"), - Value::test_string("min"), - Value::test_string("25%"), - Value::test_string("50%"), - Value::test_string("75%"), - Value::test_string("max"), - ], - ), - Column::new( - "a (i64)".to_string(), - vec![ - Value::test_float(2.0), - Value::test_float(2.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(0.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(1.0), - ], - ), - Column::new( - "b (i64)".to_string(), - vec![ - Value::test_float(2.0), - Value::test_float(2.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(0.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(1.0), - Value::test_float(1.0), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let quantiles: Option> = call.get_flag(engine_state, stack, "quantiles")?; - let quantiles = quantiles.map(|values| { - values - .iter() - .map(|value| { - let span = value.span(); - match value { - Value::Float { val, .. } => { - if (&0.0..=&1.0).contains(&val) { - Ok(*val) - } else { - Err(ShellError::GenericError { - error: "Incorrect value for quantile".into(), - msg: "value should be between 0 and 1".into(), - span: Some(span), - help: None, - inner: vec![], - }) - } - } - Value::Error { error, .. } => Err(*error.clone()), - _ => Err(ShellError::GenericError { - error: "Incorrect value for quantile".into(), - msg: "value should be a float".into(), - span: Some(span), - help: None, - inner: vec![], - }), - } - }) - .collect::, ShellError>>() - }); - - let quantiles = match quantiles { - Some(quantiles) => quantiles?, - None => vec![0.25, 0.50, 0.75], - }; - - let mut quantiles_labels = quantiles - .iter() - .map(|q| Some(format!("{}%", q * 100.0))) - .collect::>>(); - let mut labels = vec![ - Some("count".to_string()), - Some("sum".to_string()), - Some("mean".to_string()), - Some("median".to_string()), - Some("std".to_string()), - Some("min".to_string()), - ]; - labels.append(&mut quantiles_labels); - labels.push(Some("max".to_string())); - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let names = ChunkedArray::::from_slice_options("descriptor", &labels).into_series(); - - let head = std::iter::once(names); - - let tail = df - .as_ref() - .get_columns() - .iter() - .filter(|col| !matches!(col.dtype(), &DataType::Object("object", _))) - .map(|col| { - let count = col.len() as f64; - - let sum = col.sum_as_series().ok().and_then(|series| { - series - .cast(&DataType::Float64) - .ok() - .and_then(|ca| match ca.get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }) - }); - - let mean = match col.mean_as_series().get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }; - - let median = match col.median_as_series() { - Ok(v) => match v.get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }, - _ => None, - }; - - let std = match col.std_as_series(0) { - Ok(v) => match v.get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }, - _ => None, - }; - - let min = col.min_as_series().ok().and_then(|series| { - series - .cast(&DataType::Float64) - .ok() - .and_then(|ca| match ca.get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }) - }); - - let mut quantiles = quantiles - .clone() - .into_iter() - .map(|q| { - col.quantile_as_series(q, QuantileInterpolOptions::default()) - .ok() - .and_then(|ca| ca.cast(&DataType::Float64).ok()) - .and_then(|ca| match ca.get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }) - }) - .collect::>>(); - - let max = col.max_as_series().ok().and_then(|series| { - series - .cast(&DataType::Float64) - .ok() - .and_then(|ca| match ca.get(0) { - Ok(AnyValue::Float64(v)) => Some(v), - _ => None, - }) - }); - - let mut descriptors = vec![Some(count), sum, mean, median, std, min]; - descriptors.append(&mut quantiles); - descriptors.push(max); - - let name = format!("{} ({})", col.name(), col.dtype()); - ChunkedArray::::from_slice_options(&name, &descriptors).into_series() - }); - - let res = head.chain(tail).collect::>(); - - DataFrame::new(res) - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Summary {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/take.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/take.rs deleted file mode 100644 index 406dd1d624..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/take.rs +++ /dev/null @@ -1,148 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::DataType; - -#[derive(Clone)] -pub struct TakeDF; - -impl Command for TakeDF { - fn name(&self) -> &str { - "dfr take" - } - - fn usage(&self) -> &str { - "Creates new dataframe using the given indices." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "indices", - SyntaxShape::Any, - "list of indices used to take data", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Takes selected rows from dataframe", - example: r#"let df = ([[a b]; [4 1] [5 2] [4 3]] | dfr into-df); - let indices = ([0 2] | dfr into-df); - $df | dfr take $indices"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(4), Value::test_int(4)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Takes selected rows from series", - example: r#"let series = ([4 1 5 2 4 3] | dfr into-df); - let indices = ([0 2] | dfr into-df); - $series | dfr take $indices"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(4), Value::test_int(5)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let index_value: Value = call.req(engine_state, stack, 0)?; - let index_span = index_value.span(); - let index = NuDataFrame::try_from_value(index_value)?.as_series(index_span)?; - - let casted = match index.dtype() { - DataType::UInt32 | DataType::UInt64 | DataType::Int32 | DataType::Int64 => index - .cast(&DataType::UInt32) - .map_err(|e| ShellError::GenericError { - error: "Error casting index list".into(), - msg: e.to_string(), - span: Some(index_span), - help: None, - inner: vec![], - }), - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: "Series with incorrect type".into(), - span: Some(call.head), - help: Some("Consider using a Series with type int type".into()), - inner: vec![], - }), - }?; - - let indices = casted.u32().map_err(|e| ShellError::GenericError { - error: "Error casting index list".into(), - msg: e.to_string(), - span: Some(index_span), - help: None, - inner: vec![], - })?; - - NuDataFrame::try_from_pipeline(input, call.head).and_then(|df| { - df.as_ref() - .take(indices) - .map_err(|e| ShellError::GenericError { - error: "Error taking values".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - }) - .map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None)) - }) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(TakeDF {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_arrow.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_arrow.rs deleted file mode 100644 index 66f13121bf..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_arrow.rs +++ /dev/null @@ -1,79 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::prelude::{IpcWriter, SerWriter}; -use std::{fs::File, path::PathBuf}; - -#[derive(Clone)] -pub struct ToArrow; - -impl Command for ToArrow { - fn name(&self) -> &str { - "dfr to-arrow" - } - - fn usage(&self) -> &str { - "Saves dataframe to arrow file." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("file", SyntaxShape::Filepath, "file path to save dataframe") - .input_output_type(Type::Custom("dataframe".into()), Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Saves dataframe to arrow file", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-arrow test.arrow", - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let file_name: Spanned = call.req(engine_state, stack, 0)?; - - let mut df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut file = File::create(&file_name.item).map_err(|e| ShellError::GenericError { - error: "Error with file name".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - IpcWriter::new(&mut file) - .finish(df.as_mut()) - .map_err(|e| ShellError::GenericError { - error: "Error saving file".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span); - - Ok(PipelineData::Value( - Value::list(vec![file_value], call.head), - None, - )) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_avro.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_avro.rs deleted file mode 100644 index e5e5c6fae1..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_avro.rs +++ /dev/null @@ -1,109 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars_io::{ - avro::{AvroCompression, AvroWriter}, - SerWriter, -}; -use std::{fs::File, path::PathBuf}; - -#[derive(Clone)] -pub struct ToAvro; - -impl Command for ToAvro { - fn name(&self) -> &str { - "dfr to-avro" - } - - fn usage(&self) -> &str { - "Saves dataframe to avro file." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .named( - "compression", - SyntaxShape::String, - "use compression, supports deflate or snappy", - Some('c'), - ) - .required("file", SyntaxShape::Filepath, "file path to save dataframe") - .input_output_type(Type::Custom("dataframe".into()), Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Saves dataframe to avro file", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-avro test.avro", - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn get_compression(call: &Call) -> Result, ShellError> { - if let Some((compression, span)) = call - .get_flag_expr("compression") - .and_then(|e| e.as_string().map(|s| (s, e.span))) - { - match compression.as_ref() { - "snappy" => Ok(Some(AvroCompression::Snappy)), - "deflate" => Ok(Some(AvroCompression::Deflate)), - _ => Err(ShellError::IncorrectValue { - msg: "compression must be one of deflate or snappy".to_string(), - val_span: span, - call_span: span, - }), - } - } else { - Ok(None) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let file_name: Spanned = call.req(engine_state, stack, 0)?; - let compression = get_compression(call)?; - - let mut df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError { - error: "Error with file name".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - AvroWriter::new(file) - .with_compression(compression) - .finish(df.as_mut()) - .map_err(|e| ShellError::GenericError { - error: "Error saving file".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span); - - Ok(PipelineData::Value( - Value::list(vec![file_value], call.head), - None, - )) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_csv.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_csv.rs deleted file mode 100644 index d85bed5150..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_csv.rs +++ /dev/null @@ -1,125 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::prelude::{CsvWriter, SerWriter}; -use std::{fs::File, path::PathBuf}; - -#[derive(Clone)] -pub struct ToCSV; - -impl Command for ToCSV { - fn name(&self) -> &str { - "dfr to-csv" - } - - fn usage(&self) -> &str { - "Saves dataframe to CSV file." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("file", SyntaxShape::Filepath, "file path to save dataframe") - .named( - "delimiter", - SyntaxShape::String, - "file delimiter character", - Some('d'), - ) - .switch("no-header", "Indicates if file doesn't have header", None) - .input_output_type(Type::Custom("dataframe".into()), Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Saves dataframe to CSV file", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-csv test.csv", - result: None, - }, - Example { - description: "Saves dataframe to CSV file using other delimiter", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-csv test.csv --delimiter '|'", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let file_name: Spanned = call.req(engine_state, stack, 0)?; - let delimiter: Option> = call.get_flag(engine_state, stack, "delimiter")?; - let no_header: bool = call.has_flag(engine_state, stack, "no-header")?; - - let mut df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut file = File::create(&file_name.item).map_err(|e| ShellError::GenericError { - error: "Error with file name".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - let writer = CsvWriter::new(&mut file); - - let writer = if no_header { - writer.include_header(false) - } else { - writer.include_header(true) - }; - - let mut writer = match delimiter { - None => writer, - Some(d) => { - if d.item.len() != 1 { - return Err(ShellError::GenericError { - error: "Incorrect delimiter".into(), - msg: "Delimiter has to be one char".into(), - span: Some(d.span), - help: None, - inner: vec![], - }); - } else { - let delimiter = match d.item.chars().next() { - Some(d) => d as u8, - None => unreachable!(), - }; - - writer.with_separator(delimiter) - } - } - }; - - writer - .finish(df.as_mut()) - .map_err(|e| ShellError::GenericError { - error: "Error writing to file".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span); - - Ok(PipelineData::Value( - Value::list(vec![file_value], call.head), - None, - )) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_df.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_df.rs deleted file mode 100644 index d768c7a742..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_df.rs +++ /dev/null @@ -1,189 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuSchema}; -use nu_engine::command_prelude::*; - -use polars::prelude::*; - -#[derive(Clone)] -pub struct ToDataFrame; - -impl Command for ToDataFrame { - fn name(&self) -> &str { - "dfr into-df" - } - - fn usage(&self) -> &str { - "Converts a list, table or record into a dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .named( - "schema", - SyntaxShape::Record(vec![]), - r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#, - Some('s'), - ) - .input_output_type(Type::Any, Type::Custom("dataframe".into())) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Takes a dictionary and creates a dataframe", - example: "[[a b];[1 2] [3 4]] | dfr into-df", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Takes a list of tables and creates a dataframe", - example: "[[1 2 a] [3 4 b] [5 6 c]] | dfr into-df", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "0".to_string(), - vec![Value::test_int(1), Value::test_int(3), Value::test_int(5)], - ), - Column::new( - "1".to_string(), - vec![Value::test_int(2), Value::test_int(4), Value::test_int(6)], - ), - Column::new( - "2".to_string(), - vec![ - Value::test_string("a"), - Value::test_string("b"), - Value::test_string("c"), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Takes a list and creates a dataframe", - example: "[a b c] | dfr into-df", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("a"), - Value::test_string("b"), - Value::test_string("c"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Takes a list of booleans and creates a dataframe", - example: "[true true false] | dfr into-df", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Convert to a dataframe and provide a schema", - example: "{a: 1, b: {a: [1 2 3]}, c: [a b c]}| dfr into-df -s {a: u8, b: {a: list}, c: list}", - result: Some( - NuDataFrame::try_from_series(vec![ - Series::new("a", &[1u8]), - { - let dtype = DataType::Struct(vec![Field::new("a", DataType::List(Box::new(DataType::UInt64)))]); - let vals = vec![AnyValue::StructOwned( - Box::new((vec![AnyValue::List(Series::new("a", &[1u64, 2, 3]))], vec![Field::new("a", DataType::String)]))); 1]; - Series::from_any_values_and_dtype("b", &vals, &dtype, false) - .expect("Struct series should not fail") - }, - { - let dtype = DataType::List(Box::new(DataType::String)); - let vals = vec![AnyValue::List(Series::new("c", &["a", "b", "c"]))]; - Series::from_any_values_and_dtype("c", &vals, &dtype, false) - .expect("List series should not fail") - } - ], Span::test_data()) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Convert to a dataframe and provide a schema that adds a new column", - example: r#"[[a b]; [1 "foo"] [2 "bar"]] | dfr into-df -s {a: u8, b:str, c:i64} | dfr fill-null 3"#, - result: Some(NuDataFrame::try_from_series(vec![ - Series::new("a", [1u8, 2]), - Series::new("b", ["foo", "bar"]), - Series::new("c", [3i64, 3]), - ], Span::test_data()) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - } - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let maybe_schema = call - .get_flag(engine_state, stack, "schema")? - .map(|schema| NuSchema::try_from(&schema)) - .transpose()?; - - let df = NuDataFrame::try_from_iter(input.into_iter(), maybe_schema.clone())?; - - Ok(PipelineData::Value( - NuDataFrame::into_value(df, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ToDataFrame {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_json_lines.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_json_lines.rs deleted file mode 100644 index 5875f17107..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_json_lines.rs +++ /dev/null @@ -1,80 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::prelude::{JsonWriter, SerWriter}; -use std::{fs::File, io::BufWriter, path::PathBuf}; - -#[derive(Clone)] -pub struct ToJsonLines; - -impl Command for ToJsonLines { - fn name(&self) -> &str { - "dfr to-jsonl" - } - - fn usage(&self) -> &str { - "Saves dataframe to a JSON lines file." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("file", SyntaxShape::Filepath, "file path to save dataframe") - .input_output_type(Type::Custom("dataframe".into()), Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Saves dataframe to JSON lines file", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-jsonl test.jsonl", - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let file_name: Spanned = call.req(engine_state, stack, 0)?; - - let mut df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError { - error: "Error with file name".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - let buf_writer = BufWriter::new(file); - - JsonWriter::new(buf_writer) - .finish(df.as_mut()) - .map_err(|e| ShellError::GenericError { - error: "Error saving file".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span); - - Ok(PipelineData::Value( - Value::list(vec![file_value], call.head), - None, - )) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs deleted file mode 100644 index a6ab42052c..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs +++ /dev/null @@ -1,136 +0,0 @@ -use crate::dataframe::values::{NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ToNu; - -impl Command for ToNu { - fn name(&self) -> &str { - "dfr into-nu" - } - - fn usage(&self) -> &str { - "Converts a dataframe or an expression into into nushell value for access and exploration." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .named( - "rows", - SyntaxShape::Number, - "number of rows to be shown", - Some('n'), - ) - .switch("tail", "shows tail rows", Some('t')) - .input_output_types(vec![ - (Type::Custom("expression".into()), Type::Any), - (Type::Custom("dataframe".into()), Type::table()), - ]) - //.input_output_type(Type::Any, Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - let rec_1 = Value::test_record(record! { - "index" => Value::test_int(0), - "a" => Value::test_int(1), - "b" => Value::test_int(2), - }); - let rec_2 = Value::test_record(record! { - "index" => Value::test_int(1), - "a" => Value::test_int(3), - "b" => Value::test_int(4), - }); - let rec_3 = Value::test_record(record! { - "index" => Value::test_int(2), - "a" => Value::test_int(3), - "b" => Value::test_int(4), - }); - - vec![ - Example { - description: "Shows head rows from dataframe", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr into-nu", - result: Some(Value::list(vec![rec_1, rec_2], Span::test_data())), - }, - Example { - description: "Shows tail rows from dataframe", - example: "[[a b]; [1 2] [5 6] [3 4]] | dfr into-df | dfr into-nu --tail --rows 1", - result: Some(Value::list(vec![rec_3], Span::test_data())), - }, - Example { - description: "Convert a col expression into a nushell value", - example: "dfr col a | dfr into-nu", - result: Some(Value::test_record(record! { - "expr" => Value::test_string("column"), - "value" => Value::test_string("a"), - })), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - dataframe_command(engine_state, stack, call, value) - } else { - expression_command(call, value) - } - } -} - -fn dataframe_command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: Value, -) -> Result { - let rows: Option = call.get_flag(engine_state, stack, "rows")?; - let tail: bool = call.has_flag(engine_state, stack, "tail")?; - - let df = NuDataFrame::try_from_value(input)?; - - let values = if tail { - df.tail(rows, call.head)? - } else { - // if rows is specified, return those rows, otherwise return everything - if rows.is_some() { - df.head(rows, call.head)? - } else { - df.head(Some(df.height()), call.head)? - } - }; - - let value = Value::list(values, call.head); - - Ok(PipelineData::Value(value, None)) -} -fn expression_command(call: &Call, input: Value) -> Result { - let expr = NuExpression::try_from_value(input)?; - let value = expr.to_value(call.head)?; - - Ok(PipelineData::Value(value, None)) -} - -#[cfg(test)] -mod test { - use super::super::super::expressions::ExprCol; - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples_dataframe_input() { - test_dataframe(vec![Box::new(ToNu {})]) - } - - #[test] - fn test_examples_expression_input() { - test_dataframe(vec![Box::new(ToNu {}), Box::new(ExprCol {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/eager/to_parquet.rs b/crates/nu-cmd-dataframe/src/dataframe/eager/to_parquet.rs deleted file mode 100644 index ce6419a9ac..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/eager/to_parquet.rs +++ /dev/null @@ -1,79 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::prelude::ParquetWriter; -use std::{fs::File, path::PathBuf}; - -#[derive(Clone)] -pub struct ToParquet; - -impl Command for ToParquet { - fn name(&self) -> &str { - "dfr to-parquet" - } - - fn usage(&self) -> &str { - "Saves dataframe to parquet file." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("file", SyntaxShape::Filepath, "file path to save dataframe") - .input_output_type(Type::Custom("dataframe".into()), Type::Any) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Saves dataframe to parquet file", - example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-parquet test.parquet", - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let file_name: Spanned = call.req(engine_state, stack, 0)?; - - let mut df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError { - error: "Error with file name".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - ParquetWriter::new(file) - .finish(df.as_mut()) - .map_err(|e| ShellError::GenericError { - error: "Error saving file".into(), - msg: e.to_string(), - span: Some(file_name.span), - help: None, - inner: vec![], - })?; - - let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span); - - Ok(PipelineData::Value( - Value::list(vec![file_value], call.head), - None, - )) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/alias.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/alias.rs deleted file mode 100644 index 9d36100276..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/alias.rs +++ /dev/null @@ -1,86 +0,0 @@ -use crate::dataframe::values::NuExpression; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ExprAlias; - -impl Command for ExprAlias { - fn name(&self) -> &str { - "dfr as" - } - - fn usage(&self) -> &str { - "Creates an alias expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "Alias name", - SyntaxShape::String, - "Alias name for the expression", - ) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Creates and alias expression", - example: "dfr col a | dfr as new_a | dfr into-nu", - result: { - let record = Value::test_record(record! { - "expr" => Value::test_record(record! { - "expr" => Value::test_string("column"), - "value" => Value::test_string("a"), - }), - "alias" => Value::test_string("new_a"), - }); - - Some(record) - }, - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["aka", "abbr", "otherwise"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let alias: String = call.req(engine_state, stack, 0)?; - - let expr = NuExpression::try_from_pipeline(input, call.head)?; - let expr: NuExpression = expr.into_polars().alias(alias.as_str()).into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::eager::ToNu; - use crate::dataframe::expressions::ExprCol; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(ExprAlias {}), - Box::new(ExprCol {}), - Box::new(ToNu {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/arg_where.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/arg_where.rs deleted file mode 100644 index 49c13c3f44..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/arg_where.rs +++ /dev/null @@ -1,78 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -use polars::prelude::arg_where; - -#[derive(Clone)] -pub struct ExprArgWhere; - -impl Command for ExprArgWhere { - fn name(&self) -> &str { - "dfr arg-where" - } - - fn usage(&self) -> &str { - "Creates an expression that returns the arguments where expression is true." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("column name", SyntaxShape::Any, "Expression to evaluate") - .input_output_type(Type::Any, Type::Custom("expression".into())) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Return a dataframe where the value match the expression", - example: "let df = ([[a b]; [one 1] [two 2] [three 3]] | dfr into-df); - $df | dfr select (dfr arg-where ((dfr col b) >= 2) | dfr as b_arg)", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "b_arg".to_string(), - vec![Value::test_int(1), Value::test_int(2)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["condition", "match", "if"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - let value: Value = call.req(engine_state, stack, 0)?; - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = arg_where(expr.into_polars()).into(); - - Ok(PipelineData::Value(expr.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::expressions::ExprAlias; - use crate::dataframe::lazy::LazySelect; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(ExprArgWhere {}), - Box::new(ExprAlias {}), - Box::new(LazySelect {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/col.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/col.rs deleted file mode 100644 index 1520ef995d..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/col.rs +++ /dev/null @@ -1,68 +0,0 @@ -use crate::dataframe::values::NuExpression; -use nu_engine::command_prelude::*; - -use polars::prelude::col; - -#[derive(Clone)] -pub struct ExprCol; - -impl Command for ExprCol { - fn name(&self) -> &str { - "dfr col" - } - - fn usage(&self) -> &str { - "Creates a named column expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "column name", - SyntaxShape::String, - "Name of column to be used", - ) - .input_output_type(Type::Any, Type::Custom("expression".into())) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Creates a named column expression and converts it to a nu object", - example: "dfr col a | dfr into-nu", - result: Some(Value::test_record(record! { - "expr" => Value::test_string("column"), - "value" => Value::test_string("a"), - })), - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["create"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - let name: String = call.req(engine_state, stack, 0)?; - let expr: NuExpression = col(name.as_str()).into(); - - Ok(PipelineData::Value(expr.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::eager::ToNu; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ExprCol {}), Box::new(ToNu {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/concat_str.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/concat_str.rs deleted file mode 100644 index 28f9bbda71..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/concat_str.rs +++ /dev/null @@ -1,108 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -use polars::prelude::concat_str; - -#[derive(Clone)] -pub struct ExprConcatStr; - -impl Command for ExprConcatStr { - fn name(&self) -> &str { - "dfr concat-str" - } - - fn usage(&self) -> &str { - "Creates a concat string expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "separator", - SyntaxShape::String, - "Separator used during the concatenation", - ) - .required( - "concat expressions", - SyntaxShape::List(Box::new(SyntaxShape::Any)), - "Expression(s) that define the string concatenation", - ) - .input_output_type(Type::Any, Type::Custom("expression".into())) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Creates a concat string expression", - example: r#"let df = ([[a b c]; [one two 1] [three four 2]] | dfr into-df); - $df | dfr with-column ((dfr concat-str "-" [(dfr col a) (dfr col b) ((dfr col c) * 2)]) | dfr as concat)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("three")], - ), - Column::new( - "b".to_string(), - vec![Value::test_string("two"), Value::test_string("four")], - ), - Column::new( - "c".to_string(), - vec![Value::test_int(1), Value::test_int(2)], - ), - Column::new( - "concat".to_string(), - vec![ - Value::test_string("one-two-2"), - Value::test_string("three-four-4"), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["join", "connect", "update"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - let separator: String = call.req(engine_state, stack, 0)?; - let value: Value = call.req(engine_state, stack, 1)?; - - let expressions = NuExpression::extract_exprs(value)?; - let expr: NuExpression = concat_str(expressions, &separator, false).into(); - - Ok(PipelineData::Value(expr.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::eager::WithColumn; - use crate::dataframe::expressions::alias::ExprAlias; - use crate::dataframe::expressions::col::ExprCol; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(ExprConcatStr {}), - Box::new(ExprAlias {}), - Box::new(ExprCol {}), - Box::new(WithColumn {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/datepart.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/datepart.rs deleted file mode 100644 index 60913c0dc6..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/datepart.rs +++ /dev/null @@ -1,170 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use chrono::{DateTime, FixedOffset}; -use nu_engine::command_prelude::*; - -use polars::{ - datatypes::{DataType, TimeUnit}, - prelude::NamedFrom, - series::Series, -}; - -#[derive(Clone)] -pub struct ExprDatePart; - -impl Command for ExprDatePart { - fn name(&self) -> &str { - "dfr datepart" - } - - fn usage(&self) -> &str { - "Creates an expression for capturing the specified datepart in a column." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "Datepart name", - SyntaxShape::String, - "Part of the date to capture. Possible values are year, quarter, month, week, weekday, day, hour, minute, second, millisecond, microsecond, nanosecond", - ) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - let dt = DateTime::::parse_from_str( - "2021-12-30T01:02:03.123456789 +0000", - "%Y-%m-%dT%H:%M:%S.%9f %z", - ) - .expect("date calculation should not fail in test"); - vec![ - Example { - description: "Creates an expression to capture the year date part", - example: r#"[["2021-12-30T01:02:03.123456789"]] | dfr into-df | dfr as-datetime "%Y-%m-%dT%H:%M:%S.%9f" | dfr with-column [(dfr col datetime | dfr datepart year | dfr as datetime_year )]"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("datetime".to_string(), vec![Value::test_date(dt)]), - Column::new("datetime_year".to_string(), vec![Value::test_int(2021)]), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates an expression to capture multiple date parts", - example: r#"[["2021-12-30T01:02:03.123456789"]] | dfr into-df | dfr as-datetime "%Y-%m-%dT%H:%M:%S.%9f" | - dfr with-column [ (dfr col datetime | dfr datepart year | dfr as datetime_year ), - (dfr col datetime | dfr datepart month | dfr as datetime_month ), - (dfr col datetime | dfr datepart day | dfr as datetime_day ), - (dfr col datetime | dfr datepart hour | dfr as datetime_hour ), - (dfr col datetime | dfr datepart minute | dfr as datetime_minute ), - (dfr col datetime | dfr datepart second | dfr as datetime_second ), - (dfr col datetime | dfr datepart nanosecond | dfr as datetime_ns ) ]"#, - result: Some( - NuDataFrame::try_from_series( - vec![ - Series::new("datetime", &[dt.timestamp_nanos_opt()]) - .cast(&DataType::Datetime(TimeUnit::Nanoseconds, None)) - .expect("Error casting to datetime type"), - Series::new("datetime_year", &[2021_i64]), // i32 was coerced to i64 - Series::new("datetime_month", &[12_i8]), - Series::new("datetime_day", &[30_i8]), - Series::new("datetime_hour", &[1_i8]), - Series::new("datetime_minute", &[2_i8]), - Series::new("datetime_second", &[3_i8]), - Series::new("datetime_ns", &[123456789_i64]), // i32 was coerced to i64 - ], - Span::test_data(), - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn search_terms(&self) -> Vec<&str> { - vec![ - "year", - "month", - "week", - "weekday", - "quarter", - "day", - "hour", - "minute", - "second", - "millisecond", - "microsecond", - "nanosecond", - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let part: Spanned = call.req(engine_state, stack, 0)?; - - let expr = NuExpression::try_from_pipeline(input, call.head)?; - let expr_dt = expr.into_polars().dt(); - let expr = match part.item.as_str() { - "year" => expr_dt.year(), - "quarter" => expr_dt.quarter(), - "month" => expr_dt.month(), - "week" => expr_dt.week(), - "day" => expr_dt.day(), - "hour" => expr_dt.hour(), - "minute" => expr_dt.minute(), - "second" => expr_dt.second(), - "millisecond" => expr_dt.millisecond(), - "microsecond" => expr_dt.microsecond(), - "nanosecond" => expr_dt.nanosecond(), - _ => { - return Err(ShellError::UnsupportedInput { - msg: format!("{} is not a valid datepart, expected one of year, month, day, hour, minute, second, millisecond, microsecond, nanosecond", part.item), - input: "value originates from here".to_string(), - msg_span: call.head, - input_span: part.span, - }); - } - }.into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::eager::ToNu; - use crate::dataframe::eager::WithColumn; - use crate::dataframe::expressions::ExprAlias; - use crate::dataframe::expressions::ExprCol; - use crate::dataframe::series::AsDateTime; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(ExprDatePart {}), - Box::new(ExprCol {}), - Box::new(ToNu {}), - Box::new(AsDateTime {}), - Box::new(WithColumn {}), - Box::new(ExprAlias {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs deleted file mode 100644 index 4cc56e030b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/expressions_macro.rs +++ /dev/null @@ -1,736 +0,0 @@ -/// Definition of multiple Expression commands using a macro rule -/// All of these expressions have an identical body and only require -/// to have a change in the name, description and expression function -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -// The structs defined in this file are structs that form part of other commands -// since they share a similar name -macro_rules! expr_command { - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let expr = NuExpression::try_from_pipeline(input, call.head)?; - let expr: NuExpression = expr.into_polars().$func().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new($command {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]) - } - } - }; - - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident, $ddof: expr) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let expr = NuExpression::try_from_pipeline(input, call.head)?; - let expr: NuExpression = expr.into_polars().$func($ddof).into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new($command {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]) - } - } - }; -} - -// The structs defined in this file are structs that form part of other commands -// since they share a similar name -macro_rules! lazy_expr_command { - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let lazy = NuLazyFrame::try_from_value(value)?; - let lazy = NuLazyFrame::new( - lazy.from_eager, - lazy.into_polars() - .$func() - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - help: None, - span: None, - inner: vec![], - })?, - ); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().$func().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::{ - build_test_engine_state, test_dataframe_example, - }; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - // the first example should be a for the dataframe case - let example = &$command.examples()[0]; - let mut engine_state = build_test_engine_state(vec![Box::new($command {})]); - test_dataframe_example(&mut engine_state, &example) - } - - #[test] - fn test_examples_expressions() { - // the second example should be a for the dataframe case - let example = &$command.examples()[1]; - let mut engine_state = build_test_engine_state(vec![ - Box::new($command {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &example) - } - } - }; - - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident, $ddof: expr) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let lazy = NuLazyFrame::try_from_value(value)?; - let lazy = NuLazyFrame::new( - lazy.from_eager, - lazy.into_polars() - .$func($ddof) - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - help: None, - span: None, - inner: vec![], - })?, - ); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().$func($ddof).into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::{ - build_test_engine_state, test_dataframe_example, - }; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - // the first example should be a for the dataframe case - let example = &$command.examples()[0]; - let mut engine_state = build_test_engine_state(vec![Box::new($command {})]); - test_dataframe_example(&mut engine_state, &example) - } - - #[test] - fn test_examples_expressions() { - // the second example should be a for the dataframe case - let example = &$command.examples()[1]; - let mut engine_state = build_test_engine_state(vec![ - Box::new($command {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &example) - } - } - }; -} - -// ExprList command -// Expands to a command definition for a list expression -expr_command!( - ExprList, - "dfr implode", - "Aggregates a group to a Series.", - vec![Example { - description: "", - example: "", - result: None, - }], - implode, - test_implode -); - -// ExprAggGroups command -// Expands to a command definition for a agg groups expression -expr_command!( - ExprAggGroups, - "dfr agg-groups", - "Creates an agg_groups expression.", - vec![Example { - description: "", - example: "", - result: None, - }], - agg_groups, - test_groups -); - -// ExprCount command -// Expands to a command definition for a count expression -expr_command!( - ExprCount, - "dfr count", - "Creates a count expression.", - vec![Example { - description: "", - example: "", - result: None, - }], - count, - test_count -); - -// ExprNot command -// Expands to a command definition for a not expression -expr_command!( - ExprNot, - "dfr expr-not", - "Creates a not expression.", - vec![Example { - description: "Creates a not expression", - example: "(dfr col a) > 2) | dfr expr-not", - result: None, - },], - not, - test_not -); - -// ExprMax command -// Expands to a command definition for max aggregation -lazy_expr_command!( - ExprMax, - "dfr max", - "Creates a max expression or aggregates columns to their max value.", - vec![ - Example { - description: "Max value from columns in a dataframe", - example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr max", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_int(6)],), - Column::new("b".to_string(), vec![Value::test_int(4)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Max aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 4] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr max)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(4), Value::test_int(1)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ], - max, - test_max -); - -// ExprMin command -// Expands to a command definition for min aggregation -lazy_expr_command!( - ExprMin, - "dfr min", - "Creates a min expression or aggregates columns to their min value.", - vec![ - Example { - description: "Min value from columns in a dataframe", - example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr min", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_int(1)],), - Column::new("b".to_string(), vec![Value::test_int(1)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Min aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 4] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr min)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(1)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ], - min, - test_min -); - -// ExprSum command -// Expands to a command definition for sum aggregation -lazy_expr_command!( - ExprSum, - "dfr sum", - "Creates a sum expression for an aggregation or aggregates columns to their sum value.", - vec![ - Example { - description: "Sums all columns in a dataframe", - example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr sum", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_int(11)],), - Column::new("b".to_string(), vec![Value::test_int(7)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Sum aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 4] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr sum)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(6), Value::test_int(1)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ], - sum, - test_sum -); - -// ExprMean command -// Expands to a command definition for mean aggregation -lazy_expr_command!( - ExprMean, - "dfr mean", - "Creates a mean expression for an aggregation or aggregates columns to their mean value.", - vec![ - Example { - description: "Mean value from columns in a dataframe", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr mean", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_float(4.0)],), - Column::new("b".to_string(), vec![Value::test_float(2.0)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Mean aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 4] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr mean)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_float(3.0), Value::test_float(1.0)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ], - mean, - test_mean -); - -// ExprMedian command -// Expands to a command definition for median aggregation -expr_command!( - ExprMedian, - "dfr median", - "Creates a median expression for an aggregation.", - vec![Example { - description: "Median aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 4] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr median)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_float(3.0), Value::test_float(1.0)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - },], - median, - test_median -); - -// ExprStd command -// Expands to a command definition for std aggregation -lazy_expr_command!( - ExprStd, - "dfr std", - "Creates a std expression for an aggregation of std value from columns in a dataframe.", - vec![ - Example { - description: "Std value from columns in a dataframe", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr std", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_float(2.0)],), - Column::new("b".to_string(), vec![Value::test_float(0.0)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Std aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 2] [two 1] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr std)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_float(0.0), Value::test_float(0.0)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ], - std, - test_std, - 1 -); - -// ExprVar command -// Expands to a command definition for var aggregation -lazy_expr_command!( - ExprVar, - "dfr var", - "Create a var expression for an aggregation.", - vec![ - Example { - description: - "Var value from columns in a dataframe or aggregates columns to their var value", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr var", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_float(4.0)],), - Column::new("b".to_string(), vec![Value::test_float(0.0)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Var aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 2] [two 1] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr var)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_float(0.0), Value::test_float(0.0)], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ], - var, - test_var, - 1 -); diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/is_in.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/is_in.rs deleted file mode 100644 index 1579ba0e20..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/is_in.rs +++ /dev/null @@ -1,116 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -use polars::prelude::{lit, DataType}; - -#[derive(Clone)] -pub struct ExprIsIn; - -impl Command for ExprIsIn { - fn name(&self) -> &str { - "dfr is-in" - } - - fn usage(&self) -> &str { - "Creates an is-in expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "list", - SyntaxShape::List(Box::new(SyntaxShape::Any)), - "List to check if values are in", - ) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Creates a is-in expression", - example: r#"let df = ([[a b]; [one 1] [two 2] [three 3]] | dfr into-df); - $df | dfr with-column (dfr col a | dfr is-in [one two] | dfr as a_in)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![ - Value::test_string("one"), - Value::test_string("two"), - Value::test_string("three"), - ], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(1), Value::test_int(2), Value::test_int(3)], - ), - Column::new( - "a_in".to_string(), - vec![ - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(false), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["check", "contained", "is-contain", "match"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let list: Vec = call.req(engine_state, stack, 0)?; - let expr = NuExpression::try_from_pipeline(input, call.head)?; - - let values = - NuDataFrame::try_from_columns(vec![Column::new("list".to_string(), list)], None)?; - let list = values.as_series(call.head)?; - - if matches!(list.dtype(), DataType::Object(..)) { - return Err(ShellError::IncompatibleParametersSingle { - msg: "Cannot use a mixed list as argument".into(), - span: call.head, - }); - } - - let expr: NuExpression = expr.into_polars().is_in(lit(list)).into(); - Ok(PipelineData::Value(expr.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::eager::WithColumn; - use crate::dataframe::expressions::alias::ExprAlias; - use crate::dataframe::expressions::col::ExprCol; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(ExprIsIn {}), - Box::new(ExprAlias {}), - Box::new(ExprCol {}), - Box::new(WithColumn {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/lit.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/lit.rs deleted file mode 100644 index 8610a59048..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/lit.rs +++ /dev/null @@ -1,69 +0,0 @@ -use crate::dataframe::values::NuExpression; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ExprLit; - -impl Command for ExprLit { - fn name(&self) -> &str { - "dfr lit" - } - - fn usage(&self) -> &str { - "Creates a literal expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "literal", - SyntaxShape::Any, - "literal to construct the expression", - ) - .input_output_type(Type::Any, Type::Custom("expression".into())) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Created a literal expression and converts it to a nu object", - example: "dfr lit 2 | dfr into-nu", - result: Some(Value::test_record(record! { - "expr" => Value::test_string("literal"), - "value" => Value::test_string("2"), - })), - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["string", "literal", "expression"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - let literal: Value = call.req(engine_state, stack, 0)?; - - let expr = NuExpression::try_from_value(literal)?; - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::eager::ToNu; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ExprLit {}), Box::new(ToNu {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/mod.rs deleted file mode 100644 index 4ba70d900d..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/mod.rs +++ /dev/null @@ -1,62 +0,0 @@ -mod alias; -mod arg_where; -mod col; -mod concat_str; -mod datepart; -mod expressions_macro; -mod is_in; -mod lit; -mod otherwise; -mod quantile; -mod when; - -use nu_protocol::engine::StateWorkingSet; - -pub(crate) use crate::dataframe::expressions::alias::ExprAlias; -use crate::dataframe::expressions::arg_where::ExprArgWhere; -pub(super) use crate::dataframe::expressions::col::ExprCol; -pub(super) use crate::dataframe::expressions::concat_str::ExprConcatStr; -pub(crate) use crate::dataframe::expressions::datepart::ExprDatePart; -pub(crate) use crate::dataframe::expressions::expressions_macro::*; -pub(super) use crate::dataframe::expressions::is_in::ExprIsIn; -pub(super) use crate::dataframe::expressions::lit::ExprLit; -pub(super) use crate::dataframe::expressions::otherwise::ExprOtherwise; -pub(super) use crate::dataframe::expressions::quantile::ExprQuantile; -pub(super) use crate::dataframe::expressions::when::ExprWhen; - -pub fn add_expressions(working_set: &mut StateWorkingSet) { - macro_rules! bind_command { - ( $command:expr ) => { - working_set.add_decl(Box::new($command)); - }; - ( $( $command:expr ),* ) => { - $( working_set.add_decl(Box::new($command)); )* - }; - } - - // Dataframe commands - bind_command!( - ExprAlias, - ExprArgWhere, - ExprCol, - ExprConcatStr, - ExprCount, - ExprLit, - ExprWhen, - ExprOtherwise, - ExprQuantile, - ExprList, - ExprAggGroups, - ExprCount, - ExprIsIn, - ExprNot, - ExprMax, - ExprMin, - ExprSum, - ExprMean, - ExprMedian, - ExprStd, - ExprVar, - ExprDatePart - ); -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs deleted file mode 100644 index eb97c575b7..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/otherwise.rs +++ /dev/null @@ -1,126 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuWhen}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ExprOtherwise; - -impl Command for ExprOtherwise { - fn name(&self) -> &str { - "dfr otherwise" - } - - fn usage(&self) -> &str { - "Completes a when expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "otherwise expression", - SyntaxShape::Any, - "expression to apply when no when predicate matches", - ) - .input_output_type(Type::Any, Type::Custom("expression".into())) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create a when conditions", - example: "dfr when ((dfr col a) > 2) 4 | dfr otherwise 5", - result: None, - }, - Example { - description: "Create a when conditions", - example: - "dfr when ((dfr col a) > 2) 4 | dfr when ((dfr col a) < 0) 6 | dfr otherwise 0", - result: None, - }, - Example { - description: "Create a new column for the dataframe", - example: r#"[[a b]; [6 2] [1 4] [4 1]] - | dfr into-lazy - | dfr with-column ( - dfr when ((dfr col a) > 2) 4 | dfr otherwise 5 | dfr as c - ) - | dfr with-column ( - dfr when ((dfr col a) > 5) 10 | dfr when ((dfr col a) < 2) 6 | dfr otherwise 0 | dfr as d - ) - | dfr collect"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)], - ), - Column::new( - "c".to_string(), - vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)], - ), - Column::new( - "d".to_string(), - vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["condition", "else"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let otherwise_predicate: Value = call.req(engine_state, stack, 0)?; - let otherwise_predicate = NuExpression::try_from_value(otherwise_predicate)?; - - let value = input.into_value(call.head)?; - let complete: NuExpression = match NuWhen::try_from_value(value)? { - NuWhen::Then(then) => then.otherwise(otherwise_predicate.into_polars()).into(), - NuWhen::ChainedThen(chained_when) => chained_when - .otherwise(otherwise_predicate.into_polars()) - .into(), - }; - - Ok(PipelineData::Value(complete.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use crate::dataframe::eager::{ToNu, WithColumn}; - use crate::dataframe::expressions::when::ExprWhen; - use crate::dataframe::expressions::{ExprAlias, ExprCol}; - - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(WithColumn {}), - Box::new(ExprCol {}), - Box::new(ExprAlias {}), - Box::new(ExprWhen {}), - Box::new(ExprOtherwise {}), - Box::new(ToNu {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs deleted file mode 100644 index aaa1029ee9..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/quantile.rs +++ /dev/null @@ -1,101 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -use polars::prelude::{lit, QuantileInterpolOptions}; - -#[derive(Clone)] -pub struct ExprQuantile; - -impl Command for ExprQuantile { - fn name(&self) -> &str { - "dfr quantile" - } - - fn usage(&self) -> &str { - "Aggregates the columns to the selected quantile." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "quantile", - SyntaxShape::Number, - "quantile value for quantile operation", - ) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Quantile aggregation for a group-by", - example: r#"[[a b]; [one 2] [one 4] [two 1]] - | dfr into-df - | dfr group-by a - | dfr agg (dfr col b | dfr quantile 0.5)"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_string("one"), Value::test_string("two")], - ), - Column::new( - "b".to_string(), - vec![Value::test_float(4.0), Value::test_float(1.0)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["statistics", "percentile", "distribution"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - let quantile: f64 = call.req(engine_state, stack, 0)?; - - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr - .into_polars() - .quantile(lit(quantile), QuantileInterpolOptions::default()) - .into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(ExprQuantile {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs b/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs deleted file mode 100644 index 5a6aad2de7..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/expressions/when.rs +++ /dev/null @@ -1,147 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuWhen}; -use nu_engine::command_prelude::*; - -use polars::prelude::when; - -#[derive(Clone)] -pub struct ExprWhen; - -impl Command for ExprWhen { - fn name(&self) -> &str { - "dfr when" - } - - fn usage(&self) -> &str { - "Creates and modifies a when expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "when expression", - SyntaxShape::Any, - "when expression used for matching", - ) - .required( - "then expression", - SyntaxShape::Any, - "expression that will be applied when predicate is true", - ) - .input_output_type( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ) - .category(Category::Custom("expression".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create a when conditions", - example: "dfr when ((dfr col a) > 2) 4", - result: None, - }, - Example { - description: "Create a when conditions", - example: "dfr when ((dfr col a) > 2) 4 | dfr when ((dfr col a) < 0) 6", - result: None, - }, - Example { - description: "Create a new column for the dataframe", - example: r#"[[a b]; [6 2] [1 4] [4 1]] - | dfr into-lazy - | dfr with-column ( - dfr when ((dfr col a) > 2) 4 | dfr otherwise 5 | dfr as c - ) - | dfr with-column ( - dfr when ((dfr col a) > 5) 10 | dfr when ((dfr col a) < 2) 6 | dfr otherwise 0 | dfr as d - ) - | dfr collect"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(6), Value::test_int(1), Value::test_int(4)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4), Value::test_int(1)], - ), - Column::new( - "c".to_string(), - vec![Value::test_int(4), Value::test_int(5), Value::test_int(4)], - ), - Column::new( - "d".to_string(), - vec![Value::test_int(10), Value::test_int(6), Value::test_int(0)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn search_terms(&self) -> Vec<&str> { - vec!["condition", "match", "if", "else"] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let when_predicate: Value = call.req(engine_state, stack, 0)?; - let when_predicate = NuExpression::try_from_value(when_predicate)?; - - let then_predicate: Value = call.req(engine_state, stack, 1)?; - let then_predicate = NuExpression::try_from_value(then_predicate)?; - - let value = input.into_value(call.head)?; - let when_then: NuWhen = match value { - Value::Nothing { .. } => when(when_predicate.into_polars()) - .then(then_predicate.into_polars()) - .into(), - v => match NuWhen::try_from_value(v)? { - NuWhen::Then(when_then) => when_then - .when(when_predicate.into_polars()) - .then(then_predicate.into_polars()) - .into(), - NuWhen::ChainedThen(when_then_then) => when_then_then - .when(when_predicate.into_polars()) - .then(then_predicate.into_polars()) - .into(), - }, - }; - - Ok(PipelineData::Value(when_then.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use crate::dataframe::eager::{ToNu, WithColumn}; - use crate::dataframe::expressions::otherwise::ExprOtherwise; - use crate::dataframe::expressions::{ExprAlias, ExprCol}; - - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(WithColumn {}), - Box::new(ExprCol {}), - Box::new(ExprAlias {}), - Box::new(ExprWhen {}), - Box::new(ExprOtherwise {}), - Box::new(ToNu {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs deleted file mode 100644 index 715c3d156b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/aggregate.rs +++ /dev/null @@ -1,216 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame, NuLazyGroupBy}; -use nu_engine::command_prelude::*; - -use polars::{datatypes::DataType, prelude::Expr}; - -#[derive(Clone)] -pub struct LazyAggregate; - -impl Command for LazyAggregate { - fn name(&self) -> &str { - "dfr agg" - } - - fn usage(&self) -> &str { - "Performs a series of aggregations from a group-by." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest( - "Group-by expressions", - SyntaxShape::Any, - "Expression(s) that define the aggregations to be applied", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Group by and perform an aggregation", - example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]] - | dfr into-df - | dfr group-by a - | dfr agg [ - (dfr col b | dfr min | dfr as "b_min") - (dfr col b | dfr max | dfr as "b_max") - (dfr col b | dfr sum | dfr as "b_sum") - ]"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(2)], - ), - Column::new( - "b_min".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - Column::new( - "b_max".to_string(), - vec![Value::test_int(4), Value::test_int(6)], - ), - Column::new( - "b_sum".to_string(), - vec![Value::test_int(6), Value::test_int(10)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Group by and perform an aggregation", - example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]] - | dfr into-lazy - | dfr group-by a - | dfr agg [ - (dfr col b | dfr min | dfr as "b_min") - (dfr col b | dfr max | dfr as "b_max") - (dfr col b | dfr sum | dfr as "b_sum") - ] - | dfr collect"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(2)], - ), - Column::new( - "b_min".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - Column::new( - "b_max".to_string(), - vec![Value::test_int(4), Value::test_int(6)], - ), - Column::new( - "b_sum".to_string(), - vec![Value::test_int(6), Value::test_int(10)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let vals: Vec = call.rest(engine_state, stack, 0)?; - let value = Value::list(vals, call.head); - let expressions = NuExpression::extract_exprs(value)?; - - let group_by = NuLazyGroupBy::try_from_pipeline(input, call.head)?; - - if let Some(schema) = &group_by.schema { - for expr in expressions.iter() { - if let Some(name) = get_col_name(expr) { - let dtype = schema.get(name.as_str()); - - if matches!(dtype, Some(DataType::Object(..))) { - return Err(ShellError::GenericError { - error: "Object type column not supported for aggregation".into(), - msg: format!("Column '{name}' is type Object"), - span: Some(call.head), - help: Some("Aggregations cannot be performed on Object type columns. Use dtype command to check column types".into()), - inner: vec![], - }); - } - } - } - } - - let lazy = NuLazyFrame { - from_eager: group_by.from_eager, - lazy: Some(group_by.into_polars().agg(&expressions)), - schema: None, - }; - - let res = lazy.into_value(call.head)?; - Ok(PipelineData::Value(res, None)) - } -} - -fn get_col_name(expr: &Expr) -> Option { - match expr { - Expr::Column(column) => Some(column.to_string()), - Expr::Agg(agg) => match agg { - polars::prelude::AggExpr::Min { input: e, .. } - | polars::prelude::AggExpr::Max { input: e, .. } - | polars::prelude::AggExpr::Median(e) - | polars::prelude::AggExpr::NUnique(e) - | polars::prelude::AggExpr::First(e) - | polars::prelude::AggExpr::Last(e) - | polars::prelude::AggExpr::Mean(e) - | polars::prelude::AggExpr::Implode(e) - | polars::prelude::AggExpr::Count(e, _) - | polars::prelude::AggExpr::Sum(e) - | polars::prelude::AggExpr::AggGroups(e) - | polars::prelude::AggExpr::Std(e, _) - | polars::prelude::AggExpr::Var(e, _) => get_col_name(e.as_ref()), - polars::prelude::AggExpr::Quantile { expr, .. } => get_col_name(expr.as_ref()), - }, - Expr::Filter { input: expr, .. } - | Expr::Slice { input: expr, .. } - | Expr::Cast { expr, .. } - | Expr::Sort { expr, .. } - | Expr::Gather { expr, .. } - | Expr::SortBy { expr, .. } - | Expr::Exclude(expr, _) - | Expr::Alias(expr, _) - | Expr::KeepName(expr) - | Expr::Explode(expr) => get_col_name(expr.as_ref()), - Expr::Ternary { .. } - | Expr::AnonymousFunction { .. } - | Expr::Function { .. } - | Expr::Columns(_) - | Expr::DtypeColumn(_) - | Expr::Literal(_) - | Expr::BinaryExpr { .. } - | Expr::Window { .. } - | Expr::Wildcard - | Expr::RenameAlias { .. } - | Expr::Len - | Expr::Nth(_) - | Expr::SubPlan(_, _) - | Expr::Selector(_) => None, - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::expressions::{ExprAlias, ExprMax, ExprMin, ExprSum}; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - Box::new(ExprAlias {}), - Box::new(ExprMin {}), - Box::new(ExprMax {}), - Box::new(ExprSum {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/collect.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/collect.rs deleted file mode 100644 index c27591cc1d..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/collect.rs +++ /dev/null @@ -1,73 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyCollect; - -impl Command for LazyCollect { - fn name(&self) -> &str { - "dfr collect" - } - - fn usage(&self) -> &str { - "Collect lazy dataframe into eager dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "drop duplicates", - example: "[[a b]; [1 2] [3 4]] | dfr into-lazy | dfr collect", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(3)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let eager = lazy.collect(call.head)?; - let value = Value::custom(Box::new(eager), call.head); - - Ok(PipelineData::Value(value, None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyCollect {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs deleted file mode 100644 index a027e84d36..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/explode.rs +++ /dev/null @@ -1,153 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyExplode; - -impl Command for LazyExplode { - fn name(&self) -> &str { - "dfr explode" - } - - fn usage(&self) -> &str { - "Explodes a dataframe or creates a explode expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest( - "columns", - SyntaxShape::String, - "columns to explode, only applicable for dataframes", - ) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Explode the specified dataframe", - example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr explode hobbies | dfr collect", - result: Some( - NuDataFrame::try_from_columns(vec![ - Column::new( - "id".to_string(), - vec![ - Value::test_int(1), - Value::test_int(1), - Value::test_int(2), - Value::test_int(2), - ]), - Column::new( - "name".to_string(), - vec![ - Value::test_string("Mercy"), - Value::test_string("Mercy"), - Value::test_string("Bob"), - Value::test_string("Bob"), - ]), - Column::new( - "hobbies".to_string(), - vec![ - Value::test_string("Cycling"), - Value::test_string("Knitting"), - Value::test_string("Skiing"), - Value::test_string("Football"), - ]), - ], None).expect("simple df for test should not fail") - .into_value(Span::test_data()), - ) - }, - Example { - description: "Select a column and explode the values", - example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr select (dfr col hobbies | dfr explode)", - result: Some( - NuDataFrame::try_from_columns(vec![ - Column::new( - "hobbies".to_string(), - vec![ - Value::test_string("Cycling"), - Value::test_string("Knitting"), - Value::test_string("Skiing"), - Value::test_string("Football"), - ]), - ], None).expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - explode(call, input) - } -} - -pub(crate) fn explode(call: &Call, input: PipelineData) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let df = NuLazyFrame::try_from_value(value)?; - let columns: Vec = call - .positional_iter() - .filter_map(|e| e.as_string()) - .collect(); - - let exploded = df - .into_polars() - .explode(columns.iter().map(AsRef::as_ref).collect::>()); - - Ok(PipelineData::Value( - NuLazyFrame::from(exploded).into_value(call.head)?, - None, - )) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().explode().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example}; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(LazyExplode {})]); - test_dataframe_example(&mut engine_state, &LazyExplode.examples()[0]); - } - - #[ignore] - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(LazyExplode {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &LazyExplode.examples()[1]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/fetch.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/fetch.rs deleted file mode 100644 index 6ba75aa970..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/fetch.rs +++ /dev/null @@ -1,92 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyFetch; - -impl Command for LazyFetch { - fn name(&self) -> &str { - "dfr fetch" - } - - fn usage(&self) -> &str { - "Collects the lazyframe to the selected rows." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "rows", - SyntaxShape::Int, - "number of rows to be fetched from lazyframe", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Fetch a rows from the dataframe", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr fetch 2", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(6), Value::test_int(4)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(2)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let rows: i64 = call.req(engine_state, stack, 0)?; - - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let eager: NuDataFrame = lazy - .into_polars() - .fetch(rows as usize) - .map_err(|e| ShellError::GenericError { - error: "Error fetching rows".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into(); - - Ok(PipelineData::Value( - NuDataFrame::into_value(eager, call.head), - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyFetch {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs deleted file mode 100644 index 4c75f1d9a3..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_nan.rs +++ /dev/null @@ -1,143 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyFillNA; - -impl Command for LazyFillNA { - fn name(&self) -> &str { - "dfr fill-nan" - } - - fn usage(&self) -> &str { - "Replaces NaN values with the given expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "fill", - SyntaxShape::Any, - "Expression to use to fill the NAN values", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Fills the NaN values with 0", - example: "[1 2 NaN 3 NaN] | dfr into-df | dfr fill-nan 0", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_int(1), - Value::test_int(2), - Value::test_int(0), - Value::test_int(3), - Value::test_int(0), - ], - )], - None, - ) - .expect("Df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Fills the NaN values of a whole dataframe", - example: "[[a b]; [0.2 1] [0.1 NaN]] | dfr into-df | dfr fill-nan 0", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_float(0.2), Value::test_float(0.1)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(1), Value::test_int(0)], - ), - ], - None, - ) - .expect("Df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let fill: Value = call.req(engine_state, stack, 0)?; - let value = input.into_value(call.head)?; - - if NuExpression::can_downcast(&value) { - let expr = NuExpression::try_from_value(value)?; - let fill = NuExpression::try_from_value(fill)?.into_polars(); - let expr: NuExpression = expr.into_polars().fill_nan(fill).into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } else { - let val_span = value.span(); - let frame = NuDataFrame::try_from_value(value)?; - let columns = frame.columns(val_span)?; - let dataframe = columns - .into_iter() - .map(|column| { - let column_name = column.name().to_string(); - let values = column - .into_iter() - .map(|value| { - let span = value.span(); - match value { - Value::Float { val, .. } => { - if val.is_nan() { - fill.clone() - } else { - value - } - } - Value::List { vals, .. } => { - NuDataFrame::fill_list_nan(vals, span, fill.clone()) - } - _ => value, - } - }) - .collect::>(); - Column::new(column_name, values) - }) - .collect::>(); - Ok(PipelineData::Value( - NuDataFrame::try_from_columns(dataframe, None)?.into_value(call.head), - None, - )) - } - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyFillNA {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs deleted file mode 100644 index 88be2a9e88..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/fill_null.rs +++ /dev/null @@ -1,93 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyFillNull; - -impl Command for LazyFillNull { - fn name(&self) -> &str { - "dfr fill-null" - } - - fn usage(&self) -> &str { - "Replaces NULL values with the given expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "fill", - SyntaxShape::Any, - "Expression to use to fill the null values", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Fills the null values by 0", - example: "[1 2 2 3 3] | dfr into-df | dfr shift 2 | dfr fill-null 0", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_int(0), - Value::test_int(0), - Value::test_int(1), - Value::test_int(2), - Value::test_int(2), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let fill: Value = call.req(engine_state, stack, 0)?; - let value = input.into_value(call.head)?; - - if NuExpression::can_downcast(&value) { - let expr = NuExpression::try_from_value(value)?; - let fill = NuExpression::try_from_value(fill)?.into_polars(); - let expr: NuExpression = expr.into_polars().fill_null(fill).into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } else { - let lazy = NuLazyFrame::try_from_value(value)?; - let expr = NuExpression::try_from_value(fill)?.into_polars(); - let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().fill_null(expr)); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } - } -} - -#[cfg(test)] -mod test { - use super::super::super::series::Shift; - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyFillNull {}), Box::new(Shift {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/filter.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/filter.rs deleted file mode 100644 index 5635a77e88..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/filter.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyFilter; - -impl Command for LazyFilter { - fn name(&self) -> &str { - "dfr filter" - } - - fn usage(&self) -> &str { - "Filter dataframe based in expression." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "filter expression", - SyntaxShape::Any, - "Expression that define the column selection", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Filter dataframe using an expression", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr filter ((dfr col a) >= 4)", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(6), Value::test_int(4)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(2)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value: Value = call.req(engine_state, stack, 0)?; - let expression = NuExpression::try_from_value(value)?; - - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let lazy = NuLazyFrame::new( - lazy.from_eager, - lazy.into_polars().filter(expression.into_polars()), - ); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyFilter {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/flatten.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/flatten.rs deleted file mode 100644 index 602dcbcee3..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/flatten.rs +++ /dev/null @@ -1,126 +0,0 @@ -use super::explode::explode; -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazyFlatten; - -impl Command for LazyFlatten { - fn name(&self) -> &str { - "dfr flatten" - } - - fn usage(&self) -> &str { - "An alias for dfr explode." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest( - "columns", - SyntaxShape::String, - "columns to flatten, only applicable for dataframes", - ) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ -Example { - description: "Flatten the specified dataframe", - example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr flatten hobbies | dfr collect", - result: Some( - NuDataFrame::try_from_columns(vec![ - Column::new( - "id".to_string(), - vec![ - Value::test_int(1), - Value::test_int(1), - Value::test_int(2), - Value::test_int(2), - ]), - Column::new( - "name".to_string(), - vec![ - Value::test_string("Mercy"), - Value::test_string("Mercy"), - Value::test_string("Bob"), - Value::test_string("Bob"), - ]), - Column::new( - "hobbies".to_string(), - vec![ - Value::test_string("Cycling"), - Value::test_string("Knitting"), - Value::test_string("Skiing"), - Value::test_string("Football"), - ]), - ], None).expect("simple df for test should not fail") - .into_value(Span::test_data()), - ) - }, - Example { - description: "Select a column and flatten the values", - example: "[[id name hobbies]; [1 Mercy [Cycling Knitting]] [2 Bob [Skiing Football]]] | dfr into-df | dfr select (dfr col hobbies | dfr flatten)", - result: Some( - NuDataFrame::try_from_columns(vec![ - Column::new( - "hobbies".to_string(), - vec![ - Value::test_string("Cycling"), - Value::test_string("Knitting"), - Value::test_string("Skiing"), - Value::test_string("Football"), - ]), - ], None).expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - explode(call, input) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example}; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(LazyFlatten {})]); - test_dataframe_example(&mut engine_state, &LazyFlatten.examples()[0]); - } - - #[ignore] - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(LazyFlatten {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &LazyFlatten.examples()[1]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs deleted file mode 100644 index c31d563eb6..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/groupby.rs +++ /dev/null @@ -1,161 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame, NuLazyGroupBy}; -use nu_engine::command_prelude::*; - -use polars::prelude::Expr; - -#[derive(Clone)] -pub struct ToLazyGroupBy; - -impl Command for ToLazyGroupBy { - fn name(&self) -> &str { - "dfr group-by" - } - - fn usage(&self) -> &str { - "Creates a group-by object that can be used for other aggregations." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest( - "Group-by expressions", - SyntaxShape::Any, - "Expression(s) that define the lazy group-by", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Group by and perform an aggregation", - example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]] - | dfr into-df - | dfr group-by a - | dfr agg [ - (dfr col b | dfr min | dfr as "b_min") - (dfr col b | dfr max | dfr as "b_max") - (dfr col b | dfr sum | dfr as "b_sum") - ]"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(2)], - ), - Column::new( - "b_min".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - Column::new( - "b_max".to_string(), - vec![Value::test_int(4), Value::test_int(6)], - ), - Column::new( - "b_sum".to_string(), - vec![Value::test_int(6), Value::test_int(10)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Group by and perform an aggregation", - example: r#"[[a b]; [1 2] [1 4] [2 6] [2 4]] - | dfr into-lazy - | dfr group-by a - | dfr agg [ - (dfr col b | dfr min | dfr as "b_min") - (dfr col b | dfr max | dfr as "b_max") - (dfr col b | dfr sum | dfr as "b_sum") - ] - | dfr collect"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(2)], - ), - Column::new( - "b_min".to_string(), - vec![Value::test_int(2), Value::test_int(4)], - ), - Column::new( - "b_max".to_string(), - vec![Value::test_int(4), Value::test_int(6)], - ), - Column::new( - "b_sum".to_string(), - vec![Value::test_int(6), Value::test_int(10)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let vals: Vec = call.rest(engine_state, stack, 0)?; - let value = Value::list(vals, call.head); - let expressions = NuExpression::extract_exprs(value)?; - - if expressions - .iter() - .any(|expr| !matches!(expr, Expr::Column(..))) - { - let value: Value = call.req(engine_state, stack, 0)?; - return Err(ShellError::IncompatibleParametersSingle { - msg: "Expected only Col expressions".into(), - span: value.span(), - }); - } - - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let group_by = NuLazyGroupBy { - schema: lazy.schema.clone(), - from_eager: lazy.from_eager, - group_by: Some(lazy.into_polars().group_by(&expressions)), - }; - - Ok(PipelineData::Value(group_by.into_value(call.head), None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - use crate::dataframe::expressions::{ExprAlias, ExprMax, ExprMin, ExprSum}; - use crate::dataframe::lazy::aggregate::LazyAggregate; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - Box::new(ExprAlias {}), - Box::new(ExprMin {}), - Box::new(ExprMax {}), - Box::new(ExprSum {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs deleted file mode 100644 index 4ae297acfd..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/join.rs +++ /dev/null @@ -1,252 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{Expr, JoinType}; - -#[derive(Clone)] -pub struct LazyJoin; - -impl Command for LazyJoin { - fn name(&self) -> &str { - "dfr join" - } - - fn usage(&self) -> &str { - "Joins a lazy frame with other lazy frame." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("other", SyntaxShape::Any, "LazyFrame to join with") - .required("left_on", SyntaxShape::Any, "Left column(s) to join on") - .required("right_on", SyntaxShape::Any, "Right column(s) to join on") - .switch( - "inner", - "inner join between lazyframes (default)", - Some('i'), - ) - .switch("left", "left join between lazyframes", Some('l')) - .switch("outer", "outer join between lazyframes", Some('o')) - .switch("cross", "cross join between lazyframes", Some('c')) - .named( - "suffix", - SyntaxShape::String, - "Suffix to use on columns with same name", - Some('s'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Join two lazy dataframes", - example: r#"let df_a = ([[a b c];[1 "a" 0] [2 "b" 1] [1 "c" 2] [1 "c" 3]] | dfr into-lazy); - let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | dfr into-lazy); - $df_a | dfr join $df_b a foo | dfr collect"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![ - Value::test_int(1), - Value::test_int(2), - Value::test_int(1), - Value::test_int(1), - ], - ), - Column::new( - "b".to_string(), - vec![ - Value::test_string("a"), - Value::test_string("b"), - Value::test_string("c"), - Value::test_string("c"), - ], - ), - Column::new( - "c".to_string(), - vec![ - Value::test_int(0), - Value::test_int(1), - Value::test_int(2), - Value::test_int(3), - ], - ), - Column::new( - "bar".to_string(), - vec![ - Value::test_string("a"), - Value::test_string("c"), - Value::test_string("a"), - Value::test_string("a"), - ], - ), - Column::new( - "ham".to_string(), - vec![ - Value::test_string("let"), - Value::test_string("var"), - Value::test_string("let"), - Value::test_string("let"), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Join one eager dataframe with a lazy dataframe", - example: r#"let df_a = ([[a b c];[1 "a" 0] [2 "b" 1] [1 "c" 2] [1 "c" 3]] | dfr into-df); - let df_b = ([["foo" "bar" "ham"];[1 "a" "let"] [2 "c" "var"] [3 "c" "const"]] | dfr into-lazy); - $df_a | dfr join $df_b a foo"#, - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![ - Value::test_int(1), - Value::test_int(2), - Value::test_int(1), - Value::test_int(1), - ], - ), - Column::new( - "b".to_string(), - vec![ - Value::test_string("a"), - Value::test_string("b"), - Value::test_string("c"), - Value::test_string("c"), - ], - ), - Column::new( - "c".to_string(), - vec![ - Value::test_int(0), - Value::test_int(1), - Value::test_int(2), - Value::test_int(3), - ], - ), - Column::new( - "bar".to_string(), - vec![ - Value::test_string("a"), - Value::test_string("c"), - Value::test_string("a"), - Value::test_string("a"), - ], - ), - Column::new( - "ham".to_string(), - vec![ - Value::test_string("let"), - Value::test_string("var"), - Value::test_string("let"), - Value::test_string("let"), - ], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let left = call.has_flag(engine_state, stack, "left")?; - let outer = call.has_flag(engine_state, stack, "outer")?; - let cross = call.has_flag(engine_state, stack, "cross")?; - - let how = if left { - JoinType::Left - } else if outer { - JoinType::Outer { coalesce: true } - } else if cross { - JoinType::Cross - } else { - JoinType::Inner - }; - - let other: Value = call.req(engine_state, stack, 0)?; - let other = NuLazyFrame::try_from_value(other)?; - let other = other.into_polars(); - - let left_on: Value = call.req(engine_state, stack, 1)?; - let left_on = NuExpression::extract_exprs(left_on)?; - - let right_on: Value = call.req(engine_state, stack, 2)?; - let right_on = NuExpression::extract_exprs(right_on)?; - - if left_on.len() != right_on.len() { - let right_on: Value = call.req(engine_state, stack, 2)?; - return Err(ShellError::IncompatibleParametersSingle { - msg: "The right column list has a different size to the left column list".into(), - span: right_on.span(), - }); - } - - // Checking that both list of expressions are made out of col expressions or strings - for (index, list) in &[(1usize, &left_on), (2, &left_on)] { - if list.iter().any(|expr| !matches!(expr, Expr::Column(..))) { - let value: Value = call.req(engine_state, stack, *index)?; - return Err(ShellError::IncompatibleParametersSingle { - msg: "Expected only a string, col expressions or list of strings".into(), - span: value.span(), - }); - } - } - - let suffix: Option = call.get_flag(engine_state, stack, "suffix")?; - let suffix = suffix.unwrap_or_else(|| "_x".into()); - - let value = input.into_value(call.head)?; - let lazy = NuLazyFrame::try_from_value(value)?; - let from_eager = lazy.from_eager; - let lazy = lazy.into_polars(); - - let lazy = lazy - .join_builder() - .with(other) - .left_on(left_on) - .right_on(right_on) - .how(how) - .force_parallel(true) - .suffix(suffix) - .finish(); - - let lazy = NuLazyFrame::new(from_eager, lazy); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyJoin {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/macro_commands.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/macro_commands.rs deleted file mode 100644 index 89655b8e3f..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/macro_commands.rs +++ /dev/null @@ -1,246 +0,0 @@ -/// Definition of multiple lazyframe commands using a macro rule -/// All of these commands have an identical body and only require -/// to have a change in the name, description and function -use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame}; -use nu_engine::command_prelude::*; - -macro_rules! lazy_command { - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().$func()); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new($command {})]) - } - } - }; - - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident, $test: ident, $ddot: expr) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().$func($ddot)); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new($command {})]) - } - } - }; - - ($command: ident, $name: expr, $desc: expr, $examples: expr, $func: ident?, $test: ident) => { - #[derive(Clone)] - pub struct $command; - - impl Command for $command { - fn name(&self) -> &str { - $name - } - - fn usage(&self) -> &str { - $desc - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - $examples - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - - let lazy = NuLazyFrame::new( - lazy.from_eager, - lazy.into_polars() - .$func() - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - help: None, - span: None, - inner: vec![], - })?, - ); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } - } - - #[cfg(test)] - mod $test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new($command {})]) - } - } - }; -} - -// LazyReverse command -// Expands to a command definition for reverse -lazy_command!( - LazyReverse, - "dfr reverse", - "Reverses the LazyFrame", - vec![Example { - description: "Reverses the dataframe.", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr reverse", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(2), Value::test_int(4), Value::test_int(6),], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(2), Value::test_int(2), Value::test_int(2),], - ), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - },], - reverse, - test_reverse -); - -// LazyCache command -// Expands to a command definition for cache -lazy_command!( - LazyCache, - "dfr cache", - "Caches operations in a new LazyFrame.", - vec![Example { - description: "Caches the result into a new LazyFrame", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr reverse | dfr cache", - result: None, - }], - cache, - test_cache -); - -// LazyMedian command -// Expands to a command definition for median aggregation -lazy_command!( - LazyMedian, - "dfr median", - "Aggregates columns to their median value", - vec![Example { - description: "Median value from columns in a dataframe", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr median", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_float(4.0)],), - Column::new("b".to_string(), vec![Value::test_float(2.0)],), - ], - None - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - },], - median?, - test_median -); diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/mod.rs deleted file mode 100644 index cbbc4e8589..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/mod.rs +++ /dev/null @@ -1,65 +0,0 @@ -pub mod aggregate; -mod collect; -mod explode; -mod fetch; -mod fill_nan; -mod fill_null; -mod filter; -mod flatten; -pub mod groupby; -mod join; -mod macro_commands; -mod quantile; -mod select; -mod sort_by_expr; -mod to_lazy; - -use nu_protocol::engine::StateWorkingSet; - -use crate::dataframe::lazy::aggregate::LazyAggregate; -pub use crate::dataframe::lazy::collect::LazyCollect; -use crate::dataframe::lazy::fetch::LazyFetch; -use crate::dataframe::lazy::fill_nan::LazyFillNA; -pub use crate::dataframe::lazy::fill_null::LazyFillNull; -use crate::dataframe::lazy::filter::LazyFilter; -use crate::dataframe::lazy::groupby::ToLazyGroupBy; -use crate::dataframe::lazy::join::LazyJoin; -pub(crate) use crate::dataframe::lazy::macro_commands::*; -use crate::dataframe::lazy::quantile::LazyQuantile; -pub(crate) use crate::dataframe::lazy::select::LazySelect; -use crate::dataframe::lazy::sort_by_expr::LazySortBy; -pub use crate::dataframe::lazy::to_lazy::ToLazyFrame; -pub use explode::LazyExplode; -pub use flatten::LazyFlatten; - -pub fn add_lazy_decls(working_set: &mut StateWorkingSet) { - macro_rules! bind_command { - ( $command:expr ) => { - working_set.add_decl(Box::new($command)); - }; - ( $( $command:expr ),* ) => { - $( working_set.add_decl(Box::new($command)); )* - }; - } - - // Dataframe commands - bind_command!( - LazyAggregate, - LazyCache, - LazyCollect, - LazyFetch, - LazyFillNA, - LazyFillNull, - LazyFilter, - LazyJoin, - LazyQuantile, - LazyMedian, - LazyReverse, - LazySelect, - LazySortBy, - ToLazyFrame, - ToLazyGroupBy, - LazyExplode, - LazyFlatten - ); -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs deleted file mode 100644 index ac8ec590c6..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/quantile.rs +++ /dev/null @@ -1,87 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuLazyFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{lit, QuantileInterpolOptions}; - -#[derive(Clone)] -pub struct LazyQuantile; - -impl Command for LazyQuantile { - fn name(&self) -> &str { - "dfr quantile" - } - - fn usage(&self) -> &str { - "Aggregates the columns to the selected quantile." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "quantile", - SyntaxShape::Number, - "quantile value for quantile operation", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "quantile value from columns in a dataframe", - example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr quantile 0.5", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new("a".to_string(), vec![Value::test_float(4.0)]), - Column::new("b".to_string(), vec![Value::test_float(2.0)]), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - let quantile: f64 = call.req(engine_state, stack, 0)?; - - let lazy = NuLazyFrame::try_from_value(value)?; - let lazy = NuLazyFrame::new( - lazy.from_eager, - lazy.into_polars() - .quantile(lit(quantile), QuantileInterpolOptions::default()) - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - help: None, - span: None, - inner: vec![], - })?, - ); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazyQuantile {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/select.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/select.rs deleted file mode 100644 index b4f01bdc07..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/select.rs +++ /dev/null @@ -1,75 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct LazySelect; - -impl Command for LazySelect { - fn name(&self) -> &str { - "dfr select" - } - - fn usage(&self) -> &str { - "Selects columns from lazyframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest( - "select expressions", - SyntaxShape::Any, - "Expression(s) that define the column selection", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Select a column from the dataframe", - example: "[[a b]; [6 2] [4 2] [2 2]] | dfr into-df | dfr select a", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "a".to_string(), - vec![Value::test_int(6), Value::test_int(4), Value::test_int(2)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let vals: Vec = call.rest(engine_state, stack, 0)?; - let value = Value::list(vals, call.head); - let expressions = NuExpression::extract_exprs(value)?; - - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let lazy = NuLazyFrame::new(lazy.from_eager, lazy.into_polars().select(&expressions)); - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazySelect {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/sort_by_expr.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/sort_by_expr.rs deleted file mode 100644 index 2e109338a9..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/sort_by_expr.rs +++ /dev/null @@ -1,159 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; -use polars::chunked_array::ops::SortMultipleOptions; - -#[derive(Clone)] -pub struct LazySortBy; - -impl Command for LazySortBy { - fn name(&self) -> &str { - "dfr sort-by" - } - - fn usage(&self) -> &str { - "Sorts a lazy dataframe based on expression(s)." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .rest( - "sort expression", - SyntaxShape::Any, - "sort expression for the dataframe", - ) - .named( - "reverse", - SyntaxShape::List(Box::new(SyntaxShape::Boolean)), - "Reverse sorting. Default is false", - Some('r'), - ) - .switch( - "nulls-last", - "nulls are shown last in the dataframe", - Some('n'), - ) - .switch("maintain-order", "Maintains order during sort", Some('m')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Sort dataframe by one column", - example: "[[a b]; [6 2] [1 4] [4 1]] | dfr into-df | dfr sort-by a", - result: Some( - NuDataFrame::try_from_columns(vec![ - Column::new( - "a".to_string(), - vec![Value::test_int(1), Value::test_int(4), Value::test_int(6)], - ), - Column::new( - "b".to_string(), - vec![Value::test_int(4), Value::test_int(1), Value::test_int(2)], - ), - ], None) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Sort column using two columns", - example: - "[[a b]; [6 2] [1 1] [1 4] [2 4]] | dfr into-df | dfr sort-by [a b] -r [false true]", - result: Some( - NuDataFrame::try_from_columns(vec![ - Column::new( - "a".to_string(), - vec![ - Value::test_int(1), - Value::test_int(1), - Value::test_int(2), - Value::test_int(6), - ], - ), - Column::new( - "b".to_string(), - vec![ - Value::test_int(4), - Value::test_int(1), - Value::test_int(4), - Value::test_int(2), - ], - ), - ], None) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let vals: Vec = call.rest(engine_state, stack, 0)?; - let value = Value::list(vals, call.head); - let expressions = NuExpression::extract_exprs(value)?; - let nulls_last = call.has_flag(engine_state, stack, "nulls-last")?; - let maintain_order = call.has_flag(engine_state, stack, "maintain-order")?; - - let reverse: Option> = call.get_flag(engine_state, stack, "reverse")?; - let reverse = match reverse { - Some(list) => { - if expressions.len() != list.len() { - let span = call - .get_flag::(engine_state, stack, "reverse")? - .expect("already checked and it exists") - .span(); - return Err(ShellError::GenericError { - error: "Incorrect list size".into(), - msg: "Size doesn't match expression list".into(), - span: Some(span), - help: None, - inner: vec![], - }); - } else { - list - } - } - None => expressions.iter().map(|_| false).collect::>(), - }; - - let sort_options = SortMultipleOptions { - descending: reverse, - nulls_last, - multithreaded: true, - maintain_order, - }; - - let lazy = NuLazyFrame::try_from_pipeline(input, call.head)?; - let lazy = NuLazyFrame::new( - lazy.from_eager, - lazy.into_polars().sort_by_exprs(&expressions, sort_options), - ); - - Ok(PipelineData::Value( - NuLazyFrame::into_value(lazy, call.head)?, - None, - )) - } -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(LazySortBy {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/lazy/to_lazy.rs b/crates/nu-cmd-dataframe/src/dataframe/lazy/to_lazy.rs deleted file mode 100644 index 1c711cdd57..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/lazy/to_lazy.rs +++ /dev/null @@ -1,53 +0,0 @@ -use crate::dataframe::values::{NuDataFrame, NuLazyFrame, NuSchema}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct ToLazyFrame; - -impl Command for ToLazyFrame { - fn name(&self) -> &str { - "dfr into-lazy" - } - - fn usage(&self) -> &str { - "Converts a dataframe into a lazy dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .named( - "schema", - SyntaxShape::Record(vec![]), - r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#, - Some('s'), - ) - .input_output_type(Type::Any, Type::Custom("dataframe".into())) - .category(Category::Custom("lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Takes a dictionary and creates a lazy dataframe", - example: "[[a b];[1 2] [3 4]] | dfr into-lazy", - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let maybe_schema = call - .get_flag(engine_state, stack, "schema")? - .map(|schema| NuSchema::try_from(&schema)) - .transpose()?; - - let df = NuDataFrame::try_from_iter(input.into_iter(), maybe_schema)?; - let lazy = NuLazyFrame::from_dataframe(df); - let value = Value::custom(Box::new(lazy), call.head); - Ok(PipelineData::Value(value, None)) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/mod.rs deleted file mode 100644 index d99ce516be..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/mod.rs +++ /dev/null @@ -1,36 +0,0 @@ -mod eager; -mod expressions; -mod lazy; -mod series; -mod stub; -mod utils; -mod values; - -pub use eager::add_eager_decls; -pub use expressions::add_expressions; -pub use lazy::add_lazy_decls; -pub use series::add_series_decls; - -use nu_protocol::engine::{EngineState, StateWorkingSet}; - -pub fn add_dataframe_context(mut engine_state: EngineState) -> EngineState { - let delta = { - let mut working_set = StateWorkingSet::new(&engine_state); - working_set.add_decl(Box::new(stub::Dfr)); - add_series_decls(&mut working_set); - add_eager_decls(&mut working_set); - add_expressions(&mut working_set); - add_lazy_decls(&mut working_set); - - working_set.render() - }; - - if let Err(err) = engine_state.merge_delta(delta) { - eprintln!("Error creating dataframe command context: {err:?}"); - } - - engine_state -} - -#[cfg(test)] -mod test_dataframe; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/all_false.rs b/crates/nu-cmd-dataframe/src/dataframe/series/all_false.rs deleted file mode 100644 index 66921e793c..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/all_false.rs +++ /dev/null @@ -1,108 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct AllFalse; - -impl Command for AllFalse { - fn name(&self) -> &str { - "dfr all-false" - } - - fn usage(&self) -> &str { - "Returns true if all values are false." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Returns true if all values are false", - example: "[false false false] | dfr into-df | dfr all-false", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "all_false".to_string(), - vec![Value::test_bool(true)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Checks the result from a comparison", - example: r#"let s = ([5 6 2 10] | dfr into-df); - let res = ($s > 9); - $res | dfr all-false"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "all_false".to_string(), - vec![Value::test_bool(false)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let series = df.as_series(call.head)?; - let bool = series.bool().map_err(|_| ShellError::GenericError { - error: "Error converting to bool".into(), - msg: "all-false only works with series of type bool".into(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let value = Value::bool(!bool.any(), call.head); - - NuDataFrame::try_from_columns( - vec![Column::new("all_false".to_string(), vec![value])], - None, - ) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(AllFalse {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/all_true.rs b/crates/nu-cmd-dataframe/src/dataframe/series/all_true.rs deleted file mode 100644 index 16b4a9edd9..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/all_true.rs +++ /dev/null @@ -1,105 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct AllTrue; - -impl Command for AllTrue { - fn name(&self) -> &str { - "dfr all-true" - } - - fn usage(&self) -> &str { - "Returns true if all values are true." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Returns true if all values are true", - example: "[true true true] | dfr into-df | dfr all-true", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "all_true".to_string(), - vec![Value::test_bool(true)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Checks the result from a comparison", - example: r#"let s = ([5 6 2 8] | dfr into-df); - let res = ($s > 9); - $res | dfr all-true"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "all_true".to_string(), - vec![Value::test_bool(false)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let series = df.as_series(call.head)?; - let bool = series.bool().map_err(|_| ShellError::GenericError { - error: "Error converting to bool".into(), - msg: "all-false only works with series of type bool".into(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let value = Value::bool(bool.all(), call.head); - - NuDataFrame::try_from_columns(vec![Column::new("all_true".to_string(), vec![value])], None) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(AllTrue {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/arg_max.rs b/crates/nu-cmd-dataframe/src/dataframe/series/arg_max.rs deleted file mode 100644 index d7539401ab..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/arg_max.rs +++ /dev/null @@ -1,85 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{ArgAgg, IntoSeries, NewChunkedArray, UInt32Chunked}; - -#[derive(Clone)] -pub struct ArgMax; - -impl Command for ArgMax { - fn name(&self) -> &str { - "dfr arg-max" - } - - fn usage(&self) -> &str { - "Return index for max value in series." - } - - fn search_terms(&self) -> Vec<&str> { - vec!["argmax", "maximum", "most", "largest", "greatest"] - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns index for max value", - example: "[1 3 2] | dfr into-df | dfr arg-max", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new("arg_max".to_string(), vec![Value::test_int(1)])], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let res = series.arg_max(); - let chunked = match res { - Some(index) => UInt32Chunked::from_slice("arg_max", &[index as u32]), - None => UInt32Chunked::from_slice("arg_max", &[]), - }; - - let res = chunked.into_series(); - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ArgMax {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/arg_min.rs b/crates/nu-cmd-dataframe/src/dataframe/series/arg_min.rs deleted file mode 100644 index 1b685d65b4..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/arg_min.rs +++ /dev/null @@ -1,85 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{ArgAgg, IntoSeries, NewChunkedArray, UInt32Chunked}; - -#[derive(Clone)] -pub struct ArgMin; - -impl Command for ArgMin { - fn name(&self) -> &str { - "dfr arg-min" - } - - fn usage(&self) -> &str { - "Return index for min value in series." - } - - fn search_terms(&self) -> Vec<&str> { - vec!["argmin", "minimum", "least", "smallest", "lowest"] - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns index for min value", - example: "[1 3 2] | dfr into-df | dfr arg-min", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new("arg_min".to_string(), vec![Value::test_int(0)])], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let res = series.arg_min(); - let chunked = match res { - Some(index) => UInt32Chunked::from_slice("arg_min", &[index as u32]), - None => UInt32Chunked::from_slice("arg_min", &[]), - }; - - let res = chunked.into_series(); - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ArgMin {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/cumulative.rs b/crates/nu-cmd-dataframe/src/dataframe/series/cumulative.rs deleted file mode 100644 index c32875e87b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/cumulative.rs +++ /dev/null @@ -1,148 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{DataType, IntoSeries}; -use polars_ops::prelude::{cum_max, cum_min, cum_sum}; - -enum CumType { - Min, - Max, - Sum, -} - -impl CumType { - fn from_str(roll_type: &str, span: Span) -> Result { - match roll_type { - "min" => Ok(Self::Min), - "max" => Ok(Self::Max), - "sum" => Ok(Self::Sum), - _ => Err(ShellError::GenericError { - error: "Wrong operation".into(), - msg: "Operation not valid for cumulative".into(), - span: Some(span), - help: Some("Allowed values: max, min, sum".into()), - inner: vec![], - }), - } - } - - fn to_str(&self) -> &'static str { - match self { - CumType::Min => "cumulative_min", - CumType::Max => "cumulative_max", - CumType::Sum => "cumulative_sum", - } - } -} - -#[derive(Clone)] -pub struct Cumulative; - -impl Command for Cumulative { - fn name(&self) -> &str { - "dfr cumulative" - } - - fn usage(&self) -> &str { - "Cumulative calculation for a series." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("type", SyntaxShape::String, "rolling operation") - .switch("reverse", "Reverse cumulative calculation", Some('r')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Cumulative sum for a series", - example: "[1 2 3 4 5] | dfr into-df | dfr cumulative sum", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0_cumulative_sum".to_string(), - vec![ - Value::test_int(1), - Value::test_int(3), - Value::test_int(6), - Value::test_int(10), - Value::test_int(15), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let cum_type: Spanned = call.req(engine_state, stack, 0)?; - let reverse = call.has_flag(engine_state, stack, "reverse")?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - if let DataType::Object(..) = series.dtype() { - return Err(ShellError::GenericError { - error: "Found object series".into(), - msg: "Series of type object cannot be used for cumulative operation".into(), - span: Some(call.head), - help: None, - inner: vec![], - }); - } - - let cum_type = CumType::from_str(&cum_type.item, cum_type.span)?; - let mut res = match cum_type { - CumType::Max => cum_max(&series, reverse), - CumType::Min => cum_min(&series, reverse), - CumType::Sum => cum_sum(&series, reverse), - } - .map_err(|e| ShellError::GenericError { - error: "Error creating cumulative".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let name = format!("{}_{}", series.name(), cum_type.to_str()); - res.rename(&name); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Cumulative {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/as_date.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/as_date.rs deleted file mode 100644 index b406057572..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/as_date.rs +++ /dev/null @@ -1,94 +0,0 @@ -use crate::dataframe::values::NuDataFrame; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, StringMethods}; - -#[derive(Clone)] -pub struct AsDate; - -impl Command for AsDate { - fn name(&self) -> &str { - "dfr as-date" - } - - fn usage(&self) -> &str { - r#"Converts string to date."# - } - - fn extra_usage(&self) -> &str { - r#"Format example: - "%Y-%m-%d" => 2021-12-31 - "%d-%m-%Y" => 31-12-2021 - "%Y%m%d" => 2021319 (2021-03-19)"# - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("format", SyntaxShape::String, "formatting date string") - .switch("not-exact", "the format string may be contained in the date (e.g. foo-2021-01-01-bar could match 2021-01-01)", Some('n')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Converts string to date", - example: r#"["2021-12-30" "2021-12-31"] | dfr into-df | dfr as-datetime "%Y-%m-%d""#, - result: None, - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let format: String = call.req(engine_state, stack, 0)?; - let not_exact = call.has_flag(engine_state, stack, "not-exact")?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - let casted = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = if not_exact { - casted.as_date_not_exact(Some(format.as_str())) - } else { - casted.as_date(Some(format.as_str()), false) - }; - - let mut res = res - .map_err(|e| ShellError::GenericError { - error: "Error creating datetime".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - - res.rename("date"); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs deleted file mode 100644 index 6ee979b069..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/as_datetime.rs +++ /dev/null @@ -1,187 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use chrono::DateTime; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, StringMethods, TimeUnit}; - -#[derive(Clone)] -pub struct AsDateTime; - -impl Command for AsDateTime { - fn name(&self) -> &str { - "dfr as-datetime" - } - - fn usage(&self) -> &str { - r#"Converts string to datetime."# - } - - fn extra_usage(&self) -> &str { - r#"Format example: - "%y/%m/%d %H:%M:%S" => 21/12/31 12:54:98 - "%y-%m-%d %H:%M:%S" => 2021-12-31 24:58:01 - "%y/%m/%d %H:%M:%S" => 21/12/31 24:58:01 - "%y%m%d %H:%M:%S" => 210319 23:58:50 - "%Y/%m/%d %H:%M:%S" => 2021/12/31 12:54:98 - "%Y-%m-%d %H:%M:%S" => 2021-12-31 24:58:01 - "%Y/%m/%d %H:%M:%S" => 2021/12/31 24:58:01 - "%Y%m%d %H:%M:%S" => 20210319 23:58:50 - "%FT%H:%M:%S" => 2019-04-18T02:45:55 - "%FT%H:%M:%S.%6f" => microseconds - "%FT%H:%M:%S.%9f" => nanoseconds"# - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("format", SyntaxShape::String, "formatting date time string") - .switch("not-exact", "the format string may be contained in the date (e.g. foo-2021-01-01-bar could match 2021-01-01)", Some('n')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Converts string to datetime", - example: r#"["2021-12-30 00:00:00" "2021-12-31 00:00:00"] | dfr into-df | dfr as-datetime "%Y-%m-%d %H:%M:%S""#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "datetime".to_string(), - vec![ - Value::date( - DateTime::parse_from_str( - "2021-12-30 00:00:00 +0000", - "%Y-%m-%d %H:%M:%S %z", - ) - .expect("date calculation should not fail in test"), - Span::test_data(), - ), - Value::date( - DateTime::parse_from_str( - "2021-12-31 00:00:00 +0000", - "%Y-%m-%d %H:%M:%S %z", - ) - .expect("date calculation should not fail in test"), - Span::test_data(), - ), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Converts string to datetime with high resolutions", - example: r#"["2021-12-30 00:00:00.123456789" "2021-12-31 00:00:00.123456789"] | dfr into-df | dfr as-datetime "%Y-%m-%d %H:%M:%S.%9f""#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "datetime".to_string(), - vec![ - Value::date( - DateTime::parse_from_str( - "2021-12-30 00:00:00.123456789 +0000", - "%Y-%m-%d %H:%M:%S.%9f %z", - ) - .expect("date calculation should not fail in test"), - Span::test_data(), - ), - Value::date( - DateTime::parse_from_str( - "2021-12-31 00:00:00.123456789 +0000", - "%Y-%m-%d %H:%M:%S.%9f %z", - ) - .expect("date calculation should not fail in test"), - Span::test_data(), - ), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let format: String = call.req(engine_state, stack, 0)?; - let not_exact = call.has_flag(engine_state, stack, "not-exact")?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - let casted = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = if not_exact { - casted.as_datetime_not_exact( - Some(format.as_str()), - TimeUnit::Nanoseconds, - false, - None, - &Default::default(), - ) - } else { - casted.as_datetime( - Some(format.as_str()), - TimeUnit::Nanoseconds, - false, - false, - None, - &Default::default(), - ) - }; - - let mut res = res - .map_err(|e| ShellError::GenericError { - error: "Error creating datetime".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - - res.rename("datetime"); - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(AsDateTime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_day.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_day.rs deleted file mode 100644 index 9187219d7a..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_day.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetDay; - -impl Command for GetDay { - fn name(&self) -> &str { - "dfr get-day" - } - - fn usage(&self) -> &str { - "Gets day from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns day from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-day"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(4), Value::test_int(4)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.day().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetDay {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_hour.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_hour.rs deleted file mode 100644 index ba05843047..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_hour.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetHour; - -impl Command for GetHour { - fn name(&self) -> &str { - "dfr get-hour" - } - - fn usage(&self) -> &str { - "Gets hour from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns hour from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-hour"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(16), Value::test_int(16)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.hour().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetHour {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_minute.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_minute.rs deleted file mode 100644 index 902ed61d56..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_minute.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetMinute; - -impl Command for GetMinute { - fn name(&self) -> &str { - "dfr get-minute" - } - - fn usage(&self) -> &str { - "Gets minute from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns minute from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-minute"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(39), Value::test_int(39)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.minute().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetMinute {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_month.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_month.rs deleted file mode 100644 index 077d5afc1e..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_month.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetMonth; - -impl Command for GetMonth { - fn name(&self) -> &str { - "dfr get-month" - } - - fn usage(&self) -> &str { - "Gets month from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns month from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-month"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(8), Value::test_int(8)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.month().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetMonth {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_nanosecond.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_nanosecond.rs deleted file mode 100644 index 1543e31082..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_nanosecond.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetNanosecond; - -impl Command for GetNanosecond { - fn name(&self) -> &str { - "dfr get-nanosecond" - } - - fn usage(&self) -> &str { - "Gets nanosecond from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns nanosecond from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-nanosecond"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(0), Value::test_int(0)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.nanosecond().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetNanosecond {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_ordinal.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_ordinal.rs deleted file mode 100644 index b77ebbc14c..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_ordinal.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetOrdinal; - -impl Command for GetOrdinal { - fn name(&self) -> &str { - "dfr get-ordinal" - } - - fn usage(&self) -> &str { - "Gets ordinal from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns ordinal from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-ordinal"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(217), Value::test_int(217)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.ordinal().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetOrdinal {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_second.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_second.rs deleted file mode 100644 index e039bcc010..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_second.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetSecond; - -impl Command for GetSecond { - fn name(&self) -> &str { - "dfr get-second" - } - - fn usage(&self) -> &str { - "Gets second from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns second from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-second"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(18), Value::test_int(18)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.second().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetSecond {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_week.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_week.rs deleted file mode 100644 index 1a1bc2c12d..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_week.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetWeek; - -impl Command for GetWeek { - fn name(&self) -> &str { - "dfr get-week" - } - - fn usage(&self) -> &str { - "Gets week from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns week from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-week"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(32), Value::test_int(32)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.week().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetWeek {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_weekday.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_weekday.rs deleted file mode 100644 index b5cf1b3197..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_weekday.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetWeekDay; - -impl Command for GetWeekDay { - fn name(&self) -> &str { - "dfr get-weekday" - } - - fn usage(&self) -> &str { - "Gets weekday from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns weekday from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-weekday"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(2), Value::test_int(2)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.weekday().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetWeekDay {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_year.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/get_year.rs deleted file mode 100644 index 1ec3515949..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/get_year.rs +++ /dev/null @@ -1,90 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{DatetimeMethods, IntoSeries}; - -#[derive(Clone)] -pub struct GetYear; - -impl Command for GetYear { - fn name(&self) -> &str { - "dfr get-year" - } - - fn usage(&self) -> &str { - "Gets year from date." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns year from a date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr get-year"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(2020), Value::test_int(2020)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to datetime type".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = casted.year().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(GetYear {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/date/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/series/date/mod.rs deleted file mode 100644 index ed3895a172..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/date/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -mod as_date; -mod as_datetime; -mod get_day; -mod get_hour; -mod get_minute; -mod get_month; -mod get_nanosecond; -mod get_ordinal; -mod get_second; -mod get_week; -mod get_weekday; -mod get_year; - -pub use as_date::AsDate; -pub use as_datetime::AsDateTime; -pub use get_day::GetDay; -pub use get_hour::GetHour; -pub use get_minute::GetMinute; -pub use get_month::GetMonth; -pub use get_nanosecond::GetNanosecond; -pub use get_ordinal::GetOrdinal; -pub use get_second::GetSecond; -pub use get_week::GetWeek; -pub use get_weekday::GetWeekDay; -pub use get_year::GetYear; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_sort.rs b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_sort.rs deleted file mode 100644 index bf28cbac58..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_sort.rs +++ /dev/null @@ -1,130 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, SortOptions}; - -#[derive(Clone)] -pub struct ArgSort; - -impl Command for ArgSort { - fn name(&self) -> &str { - "dfr arg-sort" - } - - fn usage(&self) -> &str { - "Returns indexes for a sorted series." - } - - fn search_terms(&self) -> Vec<&str> { - vec!["argsort", "order", "arrange"] - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .switch("reverse", "reverse order", Some('r')) - .switch("nulls-last", "nulls ordered last", Some('n')) - .switch( - "maintain-order", - "maintain order on sorted items", - Some('m'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Returns indexes for a sorted series", - example: "[1 2 2 3 3] | dfr into-df | dfr arg-sort", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "arg_sort".to_string(), - vec![ - Value::test_int(0), - Value::test_int(1), - Value::test_int(2), - Value::test_int(3), - Value::test_int(4), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Returns indexes for a sorted series", - example: "[1 2 2 3 3] | dfr into-df | dfr arg-sort --reverse", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "arg_sort".to_string(), - vec![ - Value::test_int(3), - Value::test_int(4), - Value::test_int(1), - Value::test_int(2), - Value::test_int(0), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let sort_options = SortOptions { - descending: call.has_flag(engine_state, stack, "reverse")?, - nulls_last: call.has_flag(engine_state, stack, "nulls-last")?, - multithreaded: true, - maintain_order: call.has_flag(engine_state, stack, "maintain-order")?, - }; - - let mut res = df - .as_series(call.head)? - .arg_sort(sort_options) - .into_series(); - res.rename("arg_sort"); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ArgSort {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_true.rs b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_true.rs deleted file mode 100644 index 106e95f5ea..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_true.rs +++ /dev/null @@ -1,115 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{arg_where, col, IntoLazy}; - -#[derive(Clone)] -pub struct ArgTrue; - -impl Command for ArgTrue { - fn name(&self) -> &str { - "dfr arg-true" - } - - fn usage(&self) -> &str { - "Returns indexes where values are true." - } - - fn search_terms(&self) -> Vec<&str> { - vec!["argtrue", "truth", "boolean-true"] - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns indexes where values are true", - example: "[false true false] | dfr into-df | dfr arg-true", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "arg_true".to_string(), - vec![Value::test_int(1)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let columns = df.as_ref().get_column_names(); - if columns.len() > 1 { - return Err(ShellError::GenericError { - error: "Error using as series".into(), - msg: "dataframe has more than one column".into(), - span: Some(call.head), - help: None, - inner: vec![], - }); - } - - match columns.first() { - Some(column) => { - let expression = arg_where(col(column).eq(true)).alias("arg_true"); - let res = df - .as_ref() - .clone() - .lazy() - .select(&[expression]) - .collect() - .map_err(|err| ShellError::GenericError { - error: "Error creating index column".into(), - msg: err.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let value = NuDataFrame::dataframe_into_value(res, call.head); - Ok(PipelineData::Value(value, None)) - } - _ => Err(ShellError::UnsupportedInput { - msg: "Expected the dataframe to have a column".to_string(), - input: "".to_string(), - msg_span: call.head, - input_span: call.head, - }), - } -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ArgTrue {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_unique.rs b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_unique.rs deleted file mode 100644 index 6b69518cba..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/arg_unique.rs +++ /dev/null @@ -1,93 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::IntoSeries; - -#[derive(Clone)] -pub struct ArgUnique; - -impl Command for ArgUnique { - fn name(&self) -> &str { - "dfr arg-unique" - } - - fn usage(&self) -> &str { - "Returns indexes for unique values." - } - - fn search_terms(&self) -> Vec<&str> { - vec!["argunique", "distinct", "noduplicate", "unrepeated"] - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns indexes for unique values", - example: "[1 2 2 3 3] | dfr into-df | dfr arg-unique", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "arg_unique".to_string(), - vec![Value::test_int(0), Value::test_int(1), Value::test_int(3)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut res = df - .as_series(call.head)? - .arg_unique() - .map_err(|e| ShellError::GenericError { - error: "Error extracting unique values".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - res.rename("arg_unique"); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ArgUnique {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/mod.rs deleted file mode 100644 index c0af8c8653..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -mod arg_sort; -mod arg_true; -mod arg_unique; -mod set_with_idx; - -pub use arg_sort::ArgSort; -pub use arg_true::ArgTrue; -pub use arg_unique::ArgUnique; -pub use set_with_idx::SetWithIndex; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/set_with_idx.rs b/crates/nu-cmd-dataframe/src/dataframe/series/indexes/set_with_idx.rs deleted file mode 100644 index 307ef4d5c3..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/indexes/set_with_idx.rs +++ /dev/null @@ -1,213 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{ChunkSet, DataType, IntoSeries}; - -#[derive(Clone)] -pub struct SetWithIndex; - -impl Command for SetWithIndex { - fn name(&self) -> &str { - "dfr set-with-idx" - } - - fn usage(&self) -> &str { - "Sets value in the given index." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("value", SyntaxShape::Any, "value to be inserted in series") - .required_named( - "indices", - SyntaxShape::Any, - "list of indices indicating where to set the value", - Some('i'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Set value in selected rows from series", - example: r#"let series = ([4 1 5 2 4 3] | dfr into-df); - let indices = ([0 2] | dfr into-df); - $series | dfr set-with-idx 6 --indices $indices"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_int(6), - Value::test_int(1), - Value::test_int(6), - Value::test_int(2), - Value::test_int(4), - Value::test_int(3), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let value: Value = call.req(engine_state, stack, 0)?; - - let indices_value: Value = call - .get_flag(engine_state, stack, "indices")? - .expect("required named value"); - let indices_span = indices_value.span(); - let indices = NuDataFrame::try_from_value(indices_value)?.as_series(indices_span)?; - - let casted = match indices.dtype() { - DataType::UInt32 | DataType::UInt64 | DataType::Int32 | DataType::Int64 => indices - .as_ref() - .cast(&DataType::UInt32) - .map_err(|e| ShellError::GenericError { - error: "Error casting indices".into(), - msg: e.to_string(), - span: Some(indices_span), - help: None, - inner: vec![], - }), - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: "Series with incorrect type".into(), - span: Some(indices_span), - help: Some("Consider using a Series with type int type".into()), - inner: vec![], - }), - }?; - - let indices = casted - .u32() - .map_err(|e| ShellError::GenericError { - error: "Error casting indices".into(), - msg: e.to_string(), - span: Some(indices_span), - help: None, - inner: vec![], - })? - .into_iter() - .flatten(); - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let span = value.span(); - let res = match value { - Value::Int { val, .. } => { - let chunked = series.i64().map_err(|e| ShellError::GenericError { - error: "Error casting to i64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let res = chunked.scatter_single(indices, Some(val)).map_err(|e| { - ShellError::GenericError { - error: "Error setting value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - } - })?; - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - } - Value::Float { val, .. } => { - let chunked = series.f64().map_err(|e| ShellError::GenericError { - error: "Error casting to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let res = chunked.scatter_single(indices, Some(val)).map_err(|e| { - ShellError::GenericError { - error: "Error setting value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - } - })?; - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - } - Value::String { val, .. } => { - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let res = chunked - .scatter_single(indices, Some(val.as_ref())) - .map_err(|e| ShellError::GenericError { - error: "Error setting value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let mut res = res.into_series(); - res.rename("string"); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - } - _ => Err(ShellError::GenericError { - error: "Incorrect value type".into(), - msg: format!( - "this value cannot be set in a series of type '{}'", - series.dtype() - ), - span: Some(span), - help: None, - inner: vec![], - }), - }; - - res.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(SetWithIndex {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_duplicated.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_duplicated.rs deleted file mode 100644 index b28f977b47..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_duplicated.rs +++ /dev/null @@ -1,122 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::IntoSeries; - -#[derive(Clone)] -pub struct IsDuplicated; - -impl Command for IsDuplicated { - fn name(&self) -> &str { - "dfr is-duplicated" - } - - fn usage(&self) -> &str { - "Creates mask indicating duplicated values." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create mask indicating duplicated values", - example: "[5 6 6 6 8 8 8] | dfr into-df | dfr is-duplicated", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_duplicated".to_string(), - vec![ - Value::test_bool(false), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Create mask indicating duplicated rows in a dataframe", - example: - "[[a, b]; [1 2] [1 2] [3 3] [3 3] [1 1]] | dfr into-df | dfr is-duplicated", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_duplicated".to_string(), - vec![ - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut res = df - .as_ref() - .is_duplicated() - .map_err(|e| ShellError::GenericError { - error: "Error finding duplicates".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - - res.rename("is_duplicated"); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(IsDuplicated {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_in.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_in.rs deleted file mode 100644 index 0792d3fddf..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_in.rs +++ /dev/null @@ -1,104 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{is_in, IntoSeries}; - -#[derive(Clone)] -pub struct IsIn; - -impl Command for IsIn { - fn name(&self) -> &str { - "dfr is-in" - } - - fn usage(&self) -> &str { - "Checks if elements from a series are contained in right series." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("other", SyntaxShape::Any, "right series") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Checks if elements from a series are contained in right series", - example: r#"let other = ([1 3 6] | dfr into-df); - [5 6 6 6 8 8 8] | dfr into-df | dfr is-in $other"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_in".to_string(), - vec![ - Value::test_bool(false), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?.as_series(call.head)?; - - let other_value: Value = call.req(engine_state, stack, 0)?; - let other_span = other_value.span(); - let other_df = NuDataFrame::try_from_value(other_value)?; - let other = other_df.as_series(other_span)?; - - let mut res = is_in(&df, &other) - .map_err(|e| ShellError::GenericError { - error: "Error finding in other".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - - res.rename("is_in"); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(IsIn {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs deleted file mode 100644 index 4ed33ce951..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_not_null.rs +++ /dev/null @@ -1,122 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; -use polars::prelude::IntoSeries; - -#[derive(Clone)] -pub struct IsNotNull; - -impl Command for IsNotNull { - fn name(&self) -> &str { - "dfr is-not-null" - } - - fn usage(&self) -> &str { - "Creates mask where value is not null." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create mask where values are not null", - example: r#"let s = ([5 6 0 8] | dfr into-df); - let res = ($s / $s); - $res | dfr is-not-null"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_not_null".to_string(), - vec![ - Value::test_bool(true), - Value::test_bool(true), - Value::test_bool(false), - Value::test_bool(true), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates a is not null expression from a column", - example: "dfr col a | dfr is-not-null", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let df = NuDataFrame::try_from_value(value)?; - command(engine_state, stack, call, df) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().is_not_null().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let mut res = df.as_series(call.head)?.is_not_null(); - res.rename("is_not_null"); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - use crate::dataframe::test_dataframe::{build_test_engine_state, test_dataframe_example}; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(IsNotNull {})]); - test_dataframe_example(&mut engine_state, &IsNotNull.examples()[0]); - } - - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(IsNotNull {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &IsNotNull.examples()[1]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs deleted file mode 100644 index b99d48af66..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_null.rs +++ /dev/null @@ -1,122 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; -use polars::prelude::IntoSeries; - -#[derive(Clone)] -pub struct IsNull; - -impl Command for IsNull { - fn name(&self) -> &str { - "dfr is-null" - } - - fn usage(&self) -> &str { - "Creates mask where value is null." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create mask where values are null", - example: r#"let s = ([5 6 0 8] | dfr into-df); - let res = ($s / $s); - $res | dfr is-null"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_null".to_string(), - vec![ - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(true), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates a is null expression from a column", - example: "dfr col a | dfr is-null", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let df = NuDataFrame::try_from_value(value)?; - command(engine_state, stack, call, df) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().is_null().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let mut res = df.as_series(call.head)?.is_null(); - res.rename("is_null"); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - use crate::dataframe::test_dataframe::{build_test_engine_state, test_dataframe_example}; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(IsNull {})]); - test_dataframe_example(&mut engine_state, &IsNull.examples()[0]); - } - - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(IsNull {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &IsNull.examples()[1]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_unique.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_unique.rs deleted file mode 100644 index 8e313abca7..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/is_unique.rs +++ /dev/null @@ -1,121 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::IntoSeries; - -#[derive(Clone)] -pub struct IsUnique; - -impl Command for IsUnique { - fn name(&self) -> &str { - "dfr is-unique" - } - - fn usage(&self) -> &str { - "Creates mask indicating unique values." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Create mask indicating unique values", - example: "[5 6 6 6 8 8 8] | dfr into-df | dfr is-unique", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_unique".to_string(), - vec![ - Value::test_bool(true), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Create mask indicating duplicated rows in a dataframe", - example: "[[a, b]; [1 2] [1 2] [3 3] [3 3] [1 1]] | dfr into-df | dfr is-unique", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "is_unique".to_string(), - vec![ - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(false), - Value::test_bool(true), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let mut res = df - .as_ref() - .is_unique() - .map_err(|e| ShellError::GenericError { - error: "Error finding unique values".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - - res.rename("is_unique"); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(IsUnique {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/mod.rs deleted file mode 100644 index 80c98b5ef0..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -mod is_duplicated; -mod is_in; -mod is_not_null; -mod is_null; -mod is_unique; -mod not; -mod set; - -pub use is_duplicated::IsDuplicated; -pub use is_in::IsIn; -pub use is_not_null::IsNotNull; -pub use is_null::IsNull; -pub use is_unique::IsUnique; -pub use not::NotSeries; -pub use set::SetSeries; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/not.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/not.rs deleted file mode 100644 index 081a3c3b23..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/not.rs +++ /dev/null @@ -1,93 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::IntoSeries; - -use std::ops::Not; - -#[derive(Clone)] -pub struct NotSeries; - -impl Command for NotSeries { - fn name(&self) -> &str { - "dfr not" - } - - fn usage(&self) -> &str { - "Inverts boolean mask." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Inverts boolean mask", - example: "[true false true] | dfr into-df | dfr not", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_bool(false), - Value::test_bool(true), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - command(engine_state, stack, call, df) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let series = df.as_series(call.head)?; - - let bool = series.bool().map_err(|e| ShellError::GenericError { - error: "Error inverting mask".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = bool.not(); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(NotSeries {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/masks/set.rs b/crates/nu-cmd-dataframe/src/dataframe/series/masks/set.rs deleted file mode 100644 index 4dacb7117b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/masks/set.rs +++ /dev/null @@ -1,201 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{ChunkSet, DataType, IntoSeries}; - -#[derive(Clone)] -pub struct SetSeries; - -impl Command for SetSeries { - fn name(&self) -> &str { - "dfr set" - } - - fn usage(&self) -> &str { - "Sets value where given mask is true." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("value", SyntaxShape::Any, "value to be inserted in series") - .required_named( - "mask", - SyntaxShape::Any, - "mask indicating insertions", - Some('m'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Shifts the values by a given period", - example: r#"let s = ([1 2 2 3 3] | dfr into-df | dfr shift 2); - let mask = ($s | dfr is-null); - $s | dfr set 0 --mask $mask"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_int(0), - Value::test_int(0), - Value::test_int(1), - Value::test_int(2), - Value::test_int(2), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let value: Value = call.req(engine_state, stack, 0)?; - - let mask_value: Value = call - .get_flag(engine_state, stack, "mask")? - .expect("required named value"); - let mask_span = mask_value.span(); - let mask = NuDataFrame::try_from_value(mask_value)?.as_series(mask_span)?; - - let bool_mask = match mask.dtype() { - DataType::Boolean => mask.bool().map_err(|e| ShellError::GenericError { - error: "Error casting to bool".into(), - msg: e.to_string(), - span: Some(mask_span), - help: None, - inner: vec![], - }), - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: "can only use bool series as mask".into(), - span: Some(mask_span), - help: None, - inner: vec![], - }), - }?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - let span = value.span(); - let res = match value { - Value::Int { val, .. } => { - let chunked = series.i64().map_err(|e| ShellError::GenericError { - error: "Error casting to i64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let res = chunked - .set(bool_mask, Some(val)) - .map_err(|e| ShellError::GenericError { - error: "Error setting value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - } - Value::Float { val, .. } => { - let chunked = series.f64().map_err(|e| ShellError::GenericError { - error: "Error casting to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let res = chunked - .set(bool_mask, Some(val)) - .map_err(|e| ShellError::GenericError { - error: "Error setting value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - } - Value::String { val, .. } => { - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - let res = chunked.set(bool_mask, Some(val.as_ref())).map_err(|e| { - ShellError::GenericError { - error: "Error setting value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - } - })?; - - let mut res = res.into_series(); - res.rename("string"); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - } - _ => Err(ShellError::GenericError { - error: "Incorrect value type".into(), - msg: format!( - "this value cannot be set in a series of type '{}'", - series.dtype() - ), - span: Some(span), - help: None, - inner: vec![], - }), - }; - - res.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::super::super::{IsNull, Shift}; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![ - Box::new(SetSeries {}), - Box::new(IsNull {}), - Box::new(Shift {}), - ]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/series/mod.rs deleted file mode 100644 index e1b9bc1087..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/mod.rs +++ /dev/null @@ -1,95 +0,0 @@ -mod date; -pub use date::*; - -mod string; -pub use string::*; - -mod masks; -pub use masks::*; - -mod indexes; -pub use indexes::*; - -mod all_false; -mod all_true; -mod arg_max; -mod arg_min; -mod cumulative; -mod n_null; -mod n_unique; -mod rolling; -mod shift; -mod unique; -mod value_counts; - -use nu_protocol::engine::StateWorkingSet; - -pub use all_false::AllFalse; -pub use all_true::AllTrue; -pub use arg_max::ArgMax; -pub use arg_min::ArgMin; -pub use cumulative::Cumulative; -pub use n_null::NNull; -pub use n_unique::NUnique; -pub use rolling::Rolling; -pub use shift::Shift; -pub use unique::Unique; -pub use value_counts::ValueCount; - -pub fn add_series_decls(working_set: &mut StateWorkingSet) { - macro_rules! bind_command { - ( $command:expr ) => { - working_set.add_decl(Box::new($command)); - }; - ( $( $command:expr ),* ) => { - $( working_set.add_decl(Box::new($command)); )* - }; - } - - // Series commands - bind_command!( - AllFalse, - AllTrue, - ArgMax, - ArgMin, - ArgSort, - ArgTrue, - ArgUnique, - AsDate, - AsDateTime, - Concatenate, - Contains, - Cumulative, - GetDay, - GetHour, - GetMinute, - GetMonth, - GetNanosecond, - GetOrdinal, - GetSecond, - GetWeek, - GetWeekDay, - GetYear, - IsDuplicated, - IsIn, - IsNotNull, - IsNull, - IsUnique, - NNull, - NUnique, - NotSeries, - Replace, - ReplaceAll, - Rolling, - SetSeries, - SetWithIndex, - Shift, - StrLengths, - StrSlice, - StrFTime, - ToLowerCase, - ToUpperCase, - Unique, - ValueCount - ); -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/n_null.rs b/crates/nu-cmd-dataframe/src/dataframe/series/n_null.rs deleted file mode 100644 index 6c9909da07..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/n_null.rs +++ /dev/null @@ -1,82 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct NNull; - -impl Command for NNull { - fn name(&self) -> &str { - "dfr count-null" - } - - fn usage(&self) -> &str { - "Counts null values." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Counts null values", - example: r#"let s = ([1 1 0 0 3 3 4] | dfr into-df); - ($s / $s) | dfr count-null"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "count_null".to_string(), - vec![Value::test_int(2)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let res = df.as_series(call.head)?.null_count(); - let value = Value::int(res as i64, call.head); - - NuDataFrame::try_from_columns( - vec![Column::new("count_null".to_string(), vec![value])], - None, - ) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(NNull {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs b/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs deleted file mode 100644 index c6d6e829f8..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/n_unique.rs +++ /dev/null @@ -1,127 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression}; -use nu_engine::command_prelude::*; - -#[derive(Clone)] -pub struct NUnique; - -impl Command for NUnique { - fn name(&self) -> &str { - "dfr n-unique" - } - - fn usage(&self) -> &str { - "Counts unique values." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_types(vec![ - ( - Type::Custom("expression".into()), - Type::Custom("expression".into()), - ), - ( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ), - ]) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Counts unique values", - example: "[1 1 2 2 3 3 4] | dfr into-df | dfr n-unique", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "count_unique".to_string(), - vec![Value::test_int(4)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates a is n-unique expression from a column", - example: "dfr col a | dfr n-unique", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuDataFrame::can_downcast(&value) { - let df = NuDataFrame::try_from_value(value)?; - command(engine_state, stack, call, df) - } else { - let expr = NuExpression::try_from_value(value)?; - let expr: NuExpression = expr.into_polars().n_unique().into(); - - Ok(PipelineData::Value( - NuExpression::into_value(expr, call.head), - None, - )) - } - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let res = df - .as_series(call.head)? - .n_unique() - .map_err(|e| ShellError::GenericError { - error: "Error counting unique values".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let value = Value::int(res as i64, call.head); - - NuDataFrame::try_from_columns( - vec![Column::new("count_unique".to_string(), vec![value])], - None, - ) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example}; - use super::*; - use crate::dataframe::lazy::aggregate::LazyAggregate; - use crate::dataframe::lazy::groupby::ToLazyGroupBy; - - #[test] - fn test_examples_dataframe() { - let mut engine_state = build_test_engine_state(vec![Box::new(NUnique {})]); - test_dataframe_example(&mut engine_state, &NUnique.examples()[0]); - } - - #[test] - fn test_examples_expression() { - let mut engine_state = build_test_engine_state(vec![ - Box::new(NUnique {}), - Box::new(LazyAggregate {}), - Box::new(ToLazyGroupBy {}), - ]); - test_dataframe_example(&mut engine_state, &NUnique.examples()[1]); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/rolling.rs b/crates/nu-cmd-dataframe/src/dataframe/series/rolling.rs deleted file mode 100644 index b659462298..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/rolling.rs +++ /dev/null @@ -1,186 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{DataType, Duration, IntoSeries, RollingOptionsImpl, SeriesOpsTime}; - -enum RollType { - Min, - Max, - Sum, - Mean, -} - -impl RollType { - fn from_str(roll_type: &str, span: Span) -> Result { - match roll_type { - "min" => Ok(Self::Min), - "max" => Ok(Self::Max), - "sum" => Ok(Self::Sum), - "mean" => Ok(Self::Mean), - _ => Err(ShellError::GenericError { - error: "Wrong operation".into(), - msg: "Operation not valid for cumulative".into(), - span: Some(span), - help: Some("Allowed values: min, max, sum, mean".into()), - inner: vec![], - }), - } - } - - fn to_str(&self) -> &'static str { - match self { - RollType::Min => "rolling_min", - RollType::Max => "rolling_max", - RollType::Sum => "rolling_sum", - RollType::Mean => "rolling_mean", - } - } -} - -#[derive(Clone)] -pub struct Rolling; - -impl Command for Rolling { - fn name(&self) -> &str { - "dfr rolling" - } - - fn usage(&self) -> &str { - "Rolling calculation for a series." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("type", SyntaxShape::String, "rolling operation") - .required("window", SyntaxShape::Int, "Window size for rolling") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Rolling sum for a series", - example: "[1 2 3 4 5] | dfr into-df | dfr rolling sum 2 | dfr drop-nulls", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0_rolling_sum".to_string(), - vec![ - Value::test_int(3), - Value::test_int(5), - Value::test_int(7), - Value::test_int(9), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Rolling max for a series", - example: "[1 2 3 4 5] | dfr into-df | dfr rolling max 2 | dfr drop-nulls", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0_rolling_max".to_string(), - vec![ - Value::test_int(2), - Value::test_int(3), - Value::test_int(4), - Value::test_int(5), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let roll_type: Spanned = call.req(engine_state, stack, 0)?; - let window_size: i64 = call.req(engine_state, stack, 1)?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - if let DataType::Object(..) = series.dtype() { - return Err(ShellError::GenericError { - error: "Found object series".into(), - msg: "Series of type object cannot be used for rolling operation".into(), - span: Some(call.head), - help: None, - inner: vec![], - }); - } - - let roll_type = RollType::from_str(&roll_type.item, roll_type.span)?; - - let rolling_opts = RollingOptionsImpl { - window_size: Duration::new(window_size), - min_periods: window_size as usize, - weights: None, - center: false, - by: None, - closed_window: None, - tu: None, - tz: None, - fn_params: None, - }; - let res = match roll_type { - RollType::Max => series.rolling_max(rolling_opts), - RollType::Min => series.rolling_min(rolling_opts), - RollType::Sum => series.rolling_sum(rolling_opts), - RollType::Mean => series.rolling_mean(rolling_opts), - }; - - let mut res = res.map_err(|e| ShellError::GenericError { - error: "Error calculating rolling values".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let name = format!("{}_{}", series.name(), roll_type.to_str()); - res.rename(&name); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::eager::DropNulls; - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Rolling {}), Box::new(DropNulls {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs b/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs deleted file mode 100644 index 2f40cf0a45..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/shift.rs +++ /dev/null @@ -1,115 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame}; -use nu_engine::command_prelude::*; - -use polars_plan::prelude::lit; - -#[derive(Clone)] -pub struct Shift; - -impl Command for Shift { - fn name(&self) -> &str { - "dfr shift" - } - - fn usage(&self) -> &str { - "Shifts the values by a given period." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("period", SyntaxShape::Int, "shift period") - .named( - "fill", - SyntaxShape::Any, - "Expression used to fill the null values (lazy df)", - Some('f'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe or lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Shifts the values by a given period", - example: "[1 2 2 3 3] | dfr into-df | dfr shift 2 | dfr drop-nulls", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(1), Value::test_int(2), Value::test_int(2)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuLazyFrame::can_downcast(&value) { - let df = NuLazyFrame::try_from_value(value)?; - command_lazy(engine_state, stack, call, df) - } else { - let df = NuDataFrame::try_from_value(value)?; - command_eager(engine_state, stack, call, df) - } - } -} - -fn command_eager( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let period: i64 = call.req(engine_state, stack, 0)?; - let series = df.as_series(call.head)?.shift(period); - - NuDataFrame::try_from_series(vec![series], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -fn command_lazy( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - lazy: NuLazyFrame, -) -> Result { - let shift: i64 = call.req(engine_state, stack, 0)?; - let fill: Option = call.get_flag(engine_state, stack, "fill")?; - - let lazy = lazy.into_polars(); - - let lazy: NuLazyFrame = match fill { - Some(fill) => { - let expr = NuExpression::try_from_value(fill)?.into_polars(); - lazy.shift_and_fill(lit(shift), expr).into() - } - None => lazy.shift(shift).into(), - }; - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) -} - -#[cfg(test)] -mod test { - use super::super::super::eager::DropNulls; - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Shift {}), Box::new(DropNulls {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/concatenate.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/concatenate.rs deleted file mode 100644 index d7589bd3b1..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/concatenate.rs +++ /dev/null @@ -1,113 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct Concatenate; - -impl Command for Concatenate { - fn name(&self) -> &str { - "dfr concatenate" - } - - fn usage(&self) -> &str { - "Concatenates strings with other array." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "other", - SyntaxShape::Any, - "Other array with string to be concatenated", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Concatenate string", - example: r#"let other = ([za xs cd] | dfr into-df); - [abc abc abc] | dfr into-df | dfr concatenate $other"#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("abcza"), - Value::test_string("abcxs"), - Value::test_string("abccd"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - - let other: Value = call.req(engine_state, stack, 0)?; - let other_span = other.span(); - let other_df = NuDataFrame::try_from_value(other)?; - - let other_series = other_df.as_series(other_span)?; - let other_chunked = other_series.str().map_err(|e| ShellError::GenericError { - error: "The concatenate only with string columns".into(), - msg: e.to_string(), - span: Some(other_span), - help: None, - inner: vec![], - })?; - - let series = df.as_series(call.head)?; - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "The concatenate only with string columns".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let mut res = chunked.concat(other_chunked); - - res.rename(series.name()); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Concatenate {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/contains.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/contains.rs deleted file mode 100644 index 9c1d92681e..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/contains.rs +++ /dev/null @@ -1,106 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct Contains; - -impl Command for Contains { - fn name(&self) -> &str { - "dfr contains" - } - - fn usage(&self) -> &str { - "Checks if a pattern is contained in a string." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required( - "pattern", - SyntaxShape::String, - "Regex pattern to be searched", - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns boolean indicating if pattern was found", - example: "[abc acb acb] | dfr into-df | dfr contains ab", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_bool(true), - Value::test_bool(false), - Value::test_bool(false), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let pattern: String = call.req(engine_state, stack, 0)?; - - let series = df.as_series(call.head)?; - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "The contains command only with string columns".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let res = chunked - .contains(&pattern, false) - .map_err(|e| ShellError::GenericError { - error: "Error searching in series".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Contains {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/mod.rs deleted file mode 100644 index f2fa19cbaf..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -mod concatenate; -mod contains; -mod replace; -mod replace_all; -mod str_lengths; -mod str_slice; -mod strftime; -mod to_lowercase; -mod to_uppercase; - -pub use concatenate::Concatenate; -pub use contains::Contains; -pub use replace::Replace; -pub use replace_all::ReplaceAll; -pub use str_lengths::StrLengths; -pub use str_slice::StrSlice; -pub use strftime::StrFTime; -pub use to_lowercase::ToLowerCase; -pub use to_uppercase::ToUpperCase; diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/replace.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/replace.rs deleted file mode 100644 index d954e20b66..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/replace.rs +++ /dev/null @@ -1,120 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct Replace; - -impl Command for Replace { - fn name(&self) -> &str { - "dfr replace" - } - - fn usage(&self) -> &str { - "Replace the leftmost (sub)string by a regex pattern." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required_named( - "pattern", - SyntaxShape::String, - "Regex pattern to be matched", - Some('p'), - ) - .required_named( - "replace", - SyntaxShape::String, - "replacing string", - Some('r'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Replaces string", - example: "[abc abc abc] | dfr into-df | dfr replace --pattern ab --replace AB", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("ABc"), - Value::test_string("ABc"), - Value::test_string("ABc"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let pattern: String = call - .get_flag(engine_state, stack, "pattern")? - .expect("required value"); - let replace: String = call - .get_flag(engine_state, stack, "replace")? - .expect("required value"); - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "Error conversion to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let mut res = chunked - .replace(&pattern, &replace) - .map_err(|e| ShellError::GenericError { - error: "Error finding pattern other".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - res.rename(series.name()); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Replace {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/replace_all.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/replace_all.rs deleted file mode 100644 index f329cbca73..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/replace_all.rs +++ /dev/null @@ -1,121 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct ReplaceAll; - -impl Command for ReplaceAll { - fn name(&self) -> &str { - "dfr replace-all" - } - - fn usage(&self) -> &str { - "Replace all (sub)strings by a regex pattern." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required_named( - "pattern", - SyntaxShape::String, - "Regex pattern to be matched", - Some('p'), - ) - .required_named( - "replace", - SyntaxShape::String, - "replacing string", - Some('r'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Replaces string", - example: "[abac abac abac] | dfr into-df | dfr replace-all --pattern a --replace A", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("AbAc"), - Value::test_string("AbAc"), - Value::test_string("AbAc"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let pattern: String = call - .get_flag(engine_state, stack, "pattern")? - .expect("required value"); - let replace: String = call - .get_flag(engine_state, stack, "replace")? - .expect("required value"); - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "Error conversion to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - let mut res = - chunked - .replace_all(&pattern, &replace) - .map_err(|e| ShellError::GenericError { - error: "Error finding pattern other".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })?; - - res.rename(series.name()); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ReplaceAll {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/str_lengths.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/str_lengths.rs deleted file mode 100644 index 6889cef387..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/str_lengths.rs +++ /dev/null @@ -1,87 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct StrLengths; - -impl Command for StrLengths { - fn name(&self) -> &str { - "dfr str-lengths" - } - - fn usage(&self) -> &str { - "Get lengths of all strings." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Returns string lengths", - example: "[a ab abc] | dfr into-df | dfr str-lengths", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![Value::test_int(1), Value::test_int(2), Value::test_int(3)], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-lengths command can only be used with string columns".into()), - inner: vec![], - })?; - - let res = chunked.as_ref().str_len_bytes().into_series(); - - NuDataFrame::try_from_series(vec![res], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(StrLengths {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/str_slice.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/str_slice.rs deleted file mode 100644 index 6a5c8364c2..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/str_slice.rs +++ /dev/null @@ -1,136 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::{ - prelude::{IntoSeries, NamedFrom, StringNameSpaceImpl}, - series::Series, -}; - -#[derive(Clone)] -pub struct StrSlice; - -impl Command for StrSlice { - fn name(&self) -> &str { - "dfr str-slice" - } - - fn usage(&self) -> &str { - "Slices the string from the start position until the selected length." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("start", SyntaxShape::Int, "start of slice") - .named("length", SyntaxShape::Int, "optional length", Some('l')) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Creates slices from the strings", - example: "[abcded abc321 abc123] | dfr into-df | dfr str-slice 1 --length 2", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("bc"), - Value::test_string("bc"), - Value::test_string("bc"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates slices from the strings without length", - example: "[abcded abc321 abc123] | dfr into-df | dfr str-slice 1", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("bcded"), - Value::test_string("bc321"), - Value::test_string("bc123"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let start: i64 = call.req(engine_state, stack, 0)?; - let start = Series::new("", &[start]); - - let length: Option = call.get_flag(engine_state, stack, "length")?; - let length = match length { - Some(v) => Series::new("", &[v as u64]), - None => Series::new_null("", 1), - }; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let chunked = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-slice command can only be used with string columns".into()), - inner: vec![], - })?; - - let res = chunked - .str_slice(&start, &length) - .map_err(|e| ShellError::GenericError { - error: "Dataframe Error".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .with_name(series.name()); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(StrSlice {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/strftime.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/strftime.rs deleted file mode 100644 index 3cdfa84f8e..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/strftime.rs +++ /dev/null @@ -1,105 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; - -use polars::prelude::IntoSeries; - -#[derive(Clone)] -pub struct StrFTime; - -impl Command for StrFTime { - fn name(&self) -> &str { - "dfr strftime" - } - - fn usage(&self) -> &str { - "Formats date based on string rule." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .required("fmt", SyntaxShape::String, "Format rule") - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Formats date", - example: r#"let dt = ('2020-08-04T16:39:18+00:00' | into datetime --timezone 'UTC'); - let df = ([$dt $dt] | dfr into-df); - $df | dfr strftime "%Y/%m/%d""#, - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("2020/08/04"), - Value::test_string("2020/08/04"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let fmt: String = call.req(engine_state, stack, 0)?; - - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting to date".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-slice command can only be used with string columns".into()), - inner: vec![], - })?; - - let res = casted - .strftime(&fmt) - .map_err(|e| ShellError::GenericError { - error: "Error formatting datetime".into(), - msg: e.to_string(), - span: Some(call.head), - help: None, - inner: vec![], - })? - .into_series(); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(explore_refactor_IntoDatetime)] -mod test { - use super::super::super::super::super::IntoDatetime; - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(StrFTime {}), Box::new(IntoDatetime {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/to_lowercase.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/to_lowercase.rs deleted file mode 100644 index 2340437e35..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/to_lowercase.rs +++ /dev/null @@ -1,92 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct ToLowerCase; - -impl Command for ToLowerCase { - fn name(&self) -> &str { - "dfr lowercase" - } - - fn usage(&self) -> &str { - "Lowercase the strings in the column." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Modifies strings to lowercase", - example: "[Abc aBc abC] | dfr into-df | dfr lowercase", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("abc"), - Value::test_string("abc"), - Value::test_string("abc"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-slice command can only be used with string columns".into()), - inner: vec![], - })?; - - let mut res = casted.to_lowercase(); - res.rename(series.name()); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ToLowerCase {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/string/to_uppercase.rs b/crates/nu-cmd-dataframe/src/dataframe/series/string/to_uppercase.rs deleted file mode 100644 index 23378f5dc3..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/string/to_uppercase.rs +++ /dev/null @@ -1,96 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::{IntoSeries, StringNameSpaceImpl}; - -#[derive(Clone)] -pub struct ToUpperCase; - -impl Command for ToUpperCase { - fn name(&self) -> &str { - "dfr uppercase" - } - - fn usage(&self) -> &str { - "Uppercase the strings in the column." - } - - fn search_terms(&self) -> Vec<&str> { - vec!["capitalize, caps, capital"] - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Modifies strings to uppercase", - example: "[Abc aBc abC] | dfr into-df | dfr uppercase", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new( - "0".to_string(), - vec![ - Value::test_string("ABC"), - Value::test_string("ABC"), - Value::test_string("ABC"), - ], - )], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let casted = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting to string".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-slice command can only be used with string columns".into()), - inner: vec![], - })?; - - let mut res = casted.to_uppercase(); - res.rename(series.name()); - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -#[cfg(test)] -mod test { - use super::super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ToUpperCase {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs b/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs deleted file mode 100644 index 1bc2e0dc1b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/unique.rs +++ /dev/null @@ -1,146 +0,0 @@ -use crate::dataframe::{ - utils::extract_strings, - values::{Column, NuDataFrame, NuLazyFrame}, -}; -use nu_engine::command_prelude::*; - -use polars::prelude::{IntoSeries, UniqueKeepStrategy}; - -#[derive(Clone)] -pub struct Unique; - -impl Command for Unique { - fn name(&self) -> &str { - "dfr unique" - } - - fn usage(&self) -> &str { - "Returns unique values from a dataframe." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .named( - "subset", - SyntaxShape::Any, - "Subset of column(s) to use to maintain rows (lazy df)", - Some('s'), - ) - .switch( - "last", - "Keeps last unique value. Default keeps first value (lazy df)", - Some('l'), - ) - .switch( - "maintain-order", - "Keep the same order as the original DataFrame (lazy df)", - Some('k'), - ) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe or lazyframe".into())) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Returns unique values from a series", - example: "[2 2 2 2 2] | dfr into-df | dfr unique", - result: Some( - NuDataFrame::try_from_columns( - vec![Column::new("0".to_string(), vec![Value::test_int(2)])], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }, - Example { - description: "Creates a is unique expression from a column", - example: "col a | unique", - result: None, - }, - ] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let value = input.into_value(call.head)?; - if NuLazyFrame::can_downcast(&value) { - let df = NuLazyFrame::try_from_value(value)?; - command_lazy(engine_state, stack, call, df) - } else { - let df = NuDataFrame::try_from_value(value)?; - command_eager(engine_state, stack, call, df) - } - } -} - -fn command_eager( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - df: NuDataFrame, -) -> Result { - let series = df.as_series(call.head)?; - - let res = series.unique().map_err(|e| ShellError::GenericError { - error: "Error calculating unique values".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-slice command can only be used with string columns".into()), - inner: vec![], - })?; - - NuDataFrame::try_from_series(vec![res.into_series()], call.head) - .map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None)) -} - -fn command_lazy( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - lazy: NuLazyFrame, -) -> Result { - let last = call.has_flag(engine_state, stack, "last")?; - let maintain = call.has_flag(engine_state, stack, "maintain-order")?; - - let subset: Option = call.get_flag(engine_state, stack, "subset")?; - let subset = match subset { - Some(value) => Some(extract_strings(value)?), - None => None, - }; - - let strategy = if last { - UniqueKeepStrategy::Last - } else { - UniqueKeepStrategy::First - }; - - let lazy = lazy.into_polars(); - let lazy: NuLazyFrame = if maintain { - lazy.unique(subset, strategy).into() - } else { - lazy.unique_stable(subset, strategy).into() - }; - - Ok(PipelineData::Value(lazy.into_value(call.head)?, None)) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(Unique {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/series/value_counts.rs b/crates/nu-cmd-dataframe/src/dataframe/series/value_counts.rs deleted file mode 100644 index 87d3b42b3a..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/series/value_counts.rs +++ /dev/null @@ -1,95 +0,0 @@ -use crate::dataframe::values::{Column, NuDataFrame}; -use nu_engine::command_prelude::*; -use polars::prelude::SeriesMethods; - -#[derive(Clone)] -pub struct ValueCount; - -impl Command for ValueCount { - fn name(&self) -> &str { - "dfr value-counts" - } - - fn usage(&self) -> &str { - "Returns a dataframe with the counts for unique values in series." - } - - fn signature(&self) -> Signature { - Signature::build(self.name()) - .input_output_type( - Type::Custom("dataframe".into()), - Type::Custom("dataframe".into()), - ) - .category(Category::Custom("dataframe".into())) - } - - fn examples(&self) -> Vec { - vec![Example { - description: "Calculates value counts", - example: "[5 5 5 5 6 6] | dfr into-df | dfr value-counts", - result: Some( - NuDataFrame::try_from_columns( - vec![ - Column::new( - "0".to_string(), - vec![Value::test_int(5), Value::test_int(6)], - ), - Column::new( - "count".to_string(), - vec![Value::test_int(4), Value::test_int(2)], - ), - ], - None, - ) - .expect("simple df for test should not fail") - .into_value(Span::test_data()), - ), - }] - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - command(engine_state, stack, call, input) - } -} - -fn command( - _engine_state: &EngineState, - _stack: &mut Stack, - call: &Call, - input: PipelineData, -) -> Result { - let df = NuDataFrame::try_from_pipeline(input, call.head)?; - let series = df.as_series(call.head)?; - - let res = series - .value_counts(false, false) - .map_err(|e| ShellError::GenericError { - error: "Error calculating value counts values".into(), - msg: e.to_string(), - span: Some(call.head), - help: Some("The str-slice command can only be used with string columns".into()), - inner: vec![], - })?; - - Ok(PipelineData::Value( - NuDataFrame::dataframe_into_value(res, call.head), - None, - )) -} - -#[cfg(test)] -mod test { - use super::super::super::test_dataframe::test_dataframe; - use super::*; - - #[test] - fn test_examples() { - test_dataframe(vec![Box::new(ValueCount {})]) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/stub.rs b/crates/nu-cmd-dataframe/src/dataframe/stub.rs deleted file mode 100644 index dfabbe0b82..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/stub.rs +++ /dev/null @@ -1,34 +0,0 @@ -use nu_engine::{command_prelude::*, get_full_help}; - -#[derive(Clone)] -pub struct Dfr; - -impl Command for Dfr { - fn name(&self) -> &str { - "dfr" - } - - fn usage(&self) -> &str { - "Operate with data in a dataframe format." - } - - fn signature(&self) -> nu_protocol::Signature { - Signature::build("dfr") - .category(Category::Custom("dataframe".into())) - .input_output_types(vec![(Type::Nothing, Type::String)]) - } - - fn extra_usage(&self) -> &str { - "You must use one of the following subcommands. Using this command as-is will only produce this help message." - } - - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - _input: PipelineData, - ) -> Result { - Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs b/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs deleted file mode 100644 index 39c30be9dd..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/test_dataframe.rs +++ /dev/null @@ -1,98 +0,0 @@ -use super::{ - eager::{SchemaDF, ToDataFrame}, - expressions::ExprCol, - lazy::{LazyCollect, LazyFillNull, ToLazyFrame}, -}; -use nu_cmd_lang::Let; -use nu_engine::{command_prelude::*, eval_block}; -use nu_parser::parse; -use nu_protocol::{debugger::WithoutDebug, engine::StateWorkingSet}; - -pub fn test_dataframe(cmds: Vec>) { - if cmds.is_empty() { - panic!("Empty commands vector") - } - - // The first element in the cmds vector must be the one tested - let examples = cmds[0].examples(); - let mut engine_state = build_test_engine_state(cmds.clone()); - - for example in examples { - test_dataframe_example(&mut engine_state, &example); - } -} - -pub fn build_test_engine_state(cmds: Vec>) -> Box { - let mut engine_state = Box::new(EngineState::new()); - - let delta = { - // Base functions that are needed for testing - // Try to keep this working set small to keep tests running as fast as possible - let mut working_set = StateWorkingSet::new(&engine_state); - working_set.add_decl(Box::new(Let)); - working_set.add_decl(Box::new(ToDataFrame)); - working_set.add_decl(Box::new(ToLazyFrame)); - working_set.add_decl(Box::new(LazyCollect)); - working_set.add_decl(Box::new(ExprCol)); - working_set.add_decl(Box::new(SchemaDF)); - working_set.add_decl(Box::new(LazyFillNull)); - - // Adding the command that is being tested to the working set - for cmd in cmds.clone() { - working_set.add_decl(cmd); - } - - working_set.render() - }; - - engine_state - .merge_delta(delta) - .expect("Error merging delta"); - - engine_state -} - -pub fn test_dataframe_example(engine_state: &mut Box, example: &Example) { - // Skip tests that don't have results to compare to - if example.result.is_none() { - return; - } - - let start = std::time::Instant::now(); - - let (block, delta) = { - let mut working_set = StateWorkingSet::new(engine_state); - let output = parse(&mut working_set, None, example.example.as_bytes(), false); - - if let Some(err) = working_set.parse_errors.first() { - panic!("test parse error in `{}`: {:?}", example.example, err) - } - - (output, working_set.render()) - }; - - engine_state - .merge_delta(delta) - .expect("Error merging delta"); - - let mut stack = Stack::new().capture(); - - let result = - eval_block::(engine_state, &mut stack, &block, PipelineData::empty()) - .unwrap_or_else(|err| panic!("test eval error in `{}`: {:?}", example.example, err)) - .into_value(Span::test_data()) - .expect("ok value"); - - println!("input: {}", example.example); - println!("result: {result:?}"); - println!("done: {:?}", start.elapsed()); - - // Note. Value implements PartialEq for Bool, Int, Float, String and Block - // If the command you are testing requires to compare another case, then - // you need to define its equality in the Value struct - if let Some(expected) = example.result.clone() { - if result != expected { - panic!("the example result is different to expected value: {result:?} != {expected:?}") - } - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/utils.rs b/crates/nu-cmd-dataframe/src/dataframe/utils.rs deleted file mode 100644 index db99d550a9..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/utils.rs +++ /dev/null @@ -1,16 +0,0 @@ -use nu_protocol::{FromValue, ShellError, Value}; - -pub fn extract_strings(value: Value) -> Result, ShellError> { - let span = value.span(); - match ( - ::from_value(value.clone()), - as FromValue>::from_value(value), - ) { - (Ok(col), Err(_)) => Ok(vec![col]), - (Err(_), Ok(cols)) => Ok(cols), - _ => Err(ShellError::IncompatibleParametersSingle { - msg: "Expected a string or list of strings".into(), - span, - }), - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/mod.rs deleted file mode 100644 index eaed15aa4b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/mod.rs +++ /dev/null @@ -1,14 +0,0 @@ -mod nu_dataframe; -mod nu_expression; -mod nu_lazyframe; -mod nu_lazygroupby; -mod nu_schema; -mod nu_when; -pub mod utils; - -pub use nu_dataframe::{Axis, Column, NuDataFrame}; -pub use nu_expression::NuExpression; -pub use nu_lazyframe::NuLazyFrame; -pub use nu_lazygroupby::NuLazyGroupBy; -pub use nu_schema::{str_to_dtype, NuSchema}; -pub use nu_when::NuWhen; diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/between_values.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/between_values.rs deleted file mode 100644 index 74a484825a..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/between_values.rs +++ /dev/null @@ -1,884 +0,0 @@ -use super::{operations::Axis, NuDataFrame}; -use nu_protocol::{ - ast::{Boolean, Comparison, Math, Operator}, - ShellError, Span, Spanned, Value, -}; -use num::Zero; -use polars::prelude::{ - BooleanType, ChunkCompare, ChunkedArray, DataType, Float64Type, Int64Type, IntoSeries, - NumOpsDispatchChecked, PolarsError, Series, StringNameSpaceImpl, -}; -use std::ops::{Add, BitAnd, BitOr, Div, Mul, Sub}; - -pub(super) fn between_dataframes( - operator: Spanned, - left: &Value, - lhs: &NuDataFrame, - right: &Value, - rhs: &NuDataFrame, -) -> Result { - let operation_span = Span::merge(left.span(), right.span()); - match operator.item { - Operator::Math(Math::Plus) => match lhs.append_df(rhs, Axis::Row, operation_span) { - Ok(df) => Ok(df.into_value(operation_span)), - Err(e) => Err(e), - }, - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - } -} - -pub(super) fn compute_between_series( - operator: Spanned, - left: &Value, - lhs: &Series, - right: &Value, - rhs: &Series, -) -> Result { - let operation_span = Span::merge(left.span(), right.span()); - match operator.item { - Operator::Math(Math::Plus) => { - let mut res = lhs + rhs; - let name = format!("sum_{}_{}", lhs.name(), rhs.name()); - res.rename(&name); - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Math(Math::Minus) => { - let mut res = lhs - rhs; - let name = format!("sub_{}_{}", lhs.name(), rhs.name()); - res.rename(&name); - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Math(Math::Multiply) => { - let mut res = lhs * rhs; - let name = format!("mul_{}_{}", lhs.name(), rhs.name()); - res.rename(&name); - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Math(Math::Divide) => { - let res = lhs.checked_div(rhs); - match res { - Ok(mut res) => { - let name = format!("div_{}_{}", lhs.name(), rhs.name()); - res.rename(&name); - NuDataFrame::series_to_value(res, operation_span) - } - Err(e) => Err(ShellError::GenericError { - error: "Division error".into(), - msg: e.to_string(), - span: Some(right.span()), - help: None, - inner: vec![], - }), - } - } - Operator::Comparison(Comparison::Equal) => { - let name = format!("eq_{}_{}", lhs.name(), rhs.name()); - let res = compare_series(lhs, rhs, name.as_str(), right.span(), Series::equal)?; - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Comparison(Comparison::NotEqual) => { - let name = format!("neq_{}_{}", lhs.name(), rhs.name()); - let res = compare_series(lhs, rhs, name.as_str(), right.span(), Series::not_equal)?; - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Comparison(Comparison::LessThan) => { - let name = format!("lt_{}_{}", lhs.name(), rhs.name()); - let res = compare_series(lhs, rhs, name.as_str(), right.span(), Series::lt)?; - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Comparison(Comparison::LessThanOrEqual) => { - let name = format!("lte_{}_{}", lhs.name(), rhs.name()); - let res = compare_series(lhs, rhs, name.as_str(), right.span(), Series::lt_eq)?; - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Comparison(Comparison::GreaterThan) => { - let name = format!("gt_{}_{}", lhs.name(), rhs.name()); - let res = compare_series(lhs, rhs, name.as_str(), right.span(), Series::gt)?; - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Comparison(Comparison::GreaterThanOrEqual) => { - let name = format!("gte_{}_{}", lhs.name(), rhs.name()); - let res = compare_series(lhs, rhs, name.as_str(), right.span(), Series::gt_eq)?; - NuDataFrame::series_to_value(res, operation_span) - } - Operator::Boolean(Boolean::And) => match lhs.dtype() { - DataType::Boolean => { - let lhs_cast = lhs.bool(); - let rhs_cast = rhs.bool(); - - match (lhs_cast, rhs_cast) { - (Ok(l), Ok(r)) => { - let mut res = l.bitand(r).into_series(); - let name = format!("and_{}_{}", lhs.name(), rhs.name()); - res.rename(&name); - NuDataFrame::series_to_value(res, operation_span) - } - _ => Err(ShellError::GenericError { - error: "Incompatible types".into(), - msg: "unable to cast to boolean".into(), - span: Some(right.span()), - help: None, - inner: vec![], - }), - } - } - _ => Err(ShellError::IncompatibleParametersSingle { - msg: format!( - "Operation {} can only be done with boolean values", - operator.item - ), - span: operation_span, - }), - }, - Operator::Boolean(Boolean::Or) => match lhs.dtype() { - DataType::Boolean => { - let lhs_cast = lhs.bool(); - let rhs_cast = rhs.bool(); - - match (lhs_cast, rhs_cast) { - (Ok(l), Ok(r)) => { - let mut res = l.bitor(r).into_series(); - let name = format!("or_{}_{}", lhs.name(), rhs.name()); - res.rename(&name); - NuDataFrame::series_to_value(res, operation_span) - } - _ => Err(ShellError::GenericError { - error: "Incompatible types".into(), - msg: "unable to cast to boolean".into(), - span: Some(right.span()), - help: None, - inner: vec![], - }), - } - } - _ => Err(ShellError::IncompatibleParametersSingle { - msg: format!( - "Operation {} can only be done with boolean values", - operator.item - ), - span: operation_span, - }), - }, - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - } -} - -fn compare_series<'s, F>( - lhs: &'s Series, - rhs: &'s Series, - name: &'s str, - span: Span, - f: F, -) -> Result -where - F: Fn(&'s Series, &'s Series) -> Result, PolarsError>, -{ - let mut res = f(lhs, rhs) - .map_err(|e| ShellError::GenericError { - error: "Equality error".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })? - .into_series(); - - res.rename(name); - Ok(res) -} - -pub(super) fn compute_series_single_value( - operator: Spanned, - left: &Value, - lhs: &NuDataFrame, - right: &Value, -) -> Result { - if !lhs.is_series() { - return Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }); - } - - let lhs_span = left.span(); - let lhs = lhs.as_series(lhs_span)?; - - match operator.item { - Operator::Math(Math::Plus) => match &right { - Value::Int { val, .. } => { - compute_series_i64(&lhs, *val, >::add, lhs_span) - } - Value::Float { val, .. } => { - compute_series_float(&lhs, *val, >::add, lhs_span) - } - Value::String { val, .. } => add_string_to_series(&lhs, val, lhs_span), - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Math(Math::Minus) => match &right { - Value::Int { val, .. } => { - compute_series_i64(&lhs, *val, >::sub, lhs_span) - } - Value::Float { val, .. } => { - compute_series_float(&lhs, *val, >::sub, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Math(Math::Multiply) => match &right { - Value::Int { val, .. } => { - compute_series_i64(&lhs, *val, >::mul, lhs_span) - } - Value::Float { val, .. } => { - compute_series_float(&lhs, *val, >::mul, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Math(Math::Divide) => { - let span = right.span(); - match &right { - Value::Int { val, .. } => { - if *val == 0 { - Err(ShellError::DivisionByZero { span }) - } else { - compute_series_i64(&lhs, *val, >::div, lhs_span) - } - } - Value::Float { val, .. } => { - if val.is_zero() { - Err(ShellError::DivisionByZero { span }) - } else { - compute_series_float(&lhs, *val, >::div, lhs_span) - } - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - } - } - Operator::Comparison(Comparison::Equal) => match &right { - Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::equal, lhs_span), - Value::Float { val, .. } => { - compare_series_float(&lhs, *val, ChunkedArray::equal, lhs_span) - } - Value::String { val, .. } => { - let equal_pattern = format!("^{}$", fancy_regex::escape(val)); - contains_series_pat(&lhs, &equal_pattern, lhs_span) - } - Value::Date { val, .. } => { - compare_series_i64(&lhs, val.timestamp_millis(), ChunkedArray::equal, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::NotEqual) => match &right { - Value::Int { val, .. } => { - compare_series_i64(&lhs, *val, ChunkedArray::not_equal, lhs_span) - } - Value::Float { val, .. } => { - compare_series_float(&lhs, *val, ChunkedArray::not_equal, lhs_span) - } - Value::Date { val, .. } => compare_series_i64( - &lhs, - val.timestamp_millis(), - ChunkedArray::not_equal, - lhs_span, - ), - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::LessThan) => match &right { - Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::lt, lhs_span), - Value::Float { val, .. } => { - compare_series_float(&lhs, *val, ChunkedArray::lt, lhs_span) - } - Value::Date { val, .. } => { - compare_series_i64(&lhs, val.timestamp_millis(), ChunkedArray::lt, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::LessThanOrEqual) => match &right { - Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::lt_eq, lhs_span), - Value::Float { val, .. } => { - compare_series_float(&lhs, *val, ChunkedArray::lt_eq, lhs_span) - } - Value::Date { val, .. } => { - compare_series_i64(&lhs, val.timestamp_millis(), ChunkedArray::lt_eq, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::GreaterThan) => match &right { - Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::gt, lhs_span), - Value::Float { val, .. } => { - compare_series_float(&lhs, *val, ChunkedArray::gt, lhs_span) - } - Value::Date { val, .. } => { - compare_series_i64(&lhs, val.timestamp_millis(), ChunkedArray::gt, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::GreaterThanOrEqual) => match &right { - Value::Int { val, .. } => compare_series_i64(&lhs, *val, ChunkedArray::gt_eq, lhs_span), - Value::Float { val, .. } => { - compare_series_float(&lhs, *val, ChunkedArray::gt_eq, lhs_span) - } - Value::Date { val, .. } => { - compare_series_i64(&lhs, val.timestamp_millis(), ChunkedArray::gt_eq, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - // TODO: update this to do a regex match instead of a simple contains? - Operator::Comparison(Comparison::RegexMatch) => match &right { - Value::String { val, .. } => contains_series_pat(&lhs, val, lhs_span), - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::StartsWith) => match &right { - Value::String { val, .. } => { - let starts_with_pattern = format!("^{}", fancy_regex::escape(val)); - contains_series_pat(&lhs, &starts_with_pattern, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - Operator::Comparison(Comparison::EndsWith) => match &right { - Value::String { val, .. } => { - let ends_with_pattern = format!("{}$", fancy_regex::escape(val)); - contains_series_pat(&lhs, &ends_with_pattern, lhs_span) - } - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - }, - _ => Err(ShellError::OperatorMismatch { - op_span: operator.span, - lhs_ty: left.get_type().to_string(), - lhs_span: left.span(), - rhs_ty: right.get_type().to_string(), - rhs_span: right.span(), - }), - } -} - -fn compute_series_i64(series: &Series, val: i64, f: F, span: Span) -> Result -where - F: Fn(ChunkedArray, i64) -> ChunkedArray, -{ - match series.dtype() { - DataType::UInt32 | DataType::Int32 | DataType::UInt64 => { - let to_i64 = series.cast(&DataType::Int64); - - match to_i64 { - Ok(series) => { - let casted = series.i64(); - compute_casted_i64(casted, val, f, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to i64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - DataType::Int64 => { - let casted = series.i64(); - compute_casted_i64(casted, val, f, span) - } - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: format!( - "Series of type {} can not be used for operations with an i64 value", - series.dtype() - ), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compute_casted_i64( - casted: Result<&ChunkedArray, PolarsError>, - val: i64, - f: F, - span: Span, -) -> Result -where - F: Fn(ChunkedArray, i64) -> ChunkedArray, -{ - match casted { - Ok(casted) => { - let res = f(casted.clone(), val); - let res = res.into_series(); - NuDataFrame::series_to_value(res, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to i64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compute_series_float(series: &Series, val: f64, f: F, span: Span) -> Result -where - F: Fn(ChunkedArray, f64) -> ChunkedArray, -{ - match series.dtype() { - DataType::Float32 => { - let to_f64 = series.cast(&DataType::Float64); - - match to_f64 { - Ok(series) => { - let casted = series.f64(); - compute_casted_f64(casted, val, f, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - DataType::Float64 => { - let casted = series.f64(); - compute_casted_f64(casted, val, f, span) - } - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: format!( - "Series of type {} can not be used for operations with a float value", - series.dtype() - ), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compute_casted_f64( - casted: Result<&ChunkedArray, PolarsError>, - val: f64, - f: F, - span: Span, -) -> Result -where - F: Fn(ChunkedArray, f64) -> ChunkedArray, -{ - match casted { - Ok(casted) => { - let res = f(casted.clone(), val); - let res = res.into_series(); - NuDataFrame::series_to_value(res, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compare_series_i64(series: &Series, val: i64, f: F, span: Span) -> Result -where - F: Fn(&ChunkedArray, i64) -> ChunkedArray, -{ - match series.dtype() { - DataType::UInt32 | DataType::Int32 | DataType::UInt64 | DataType::Datetime(_, _) => { - let to_i64 = series.cast(&DataType::Int64); - - match to_i64 { - Ok(series) => { - let casted = series.i64(); - compare_casted_i64(casted, val, f, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - DataType::Date => { - let to_i64 = series.cast(&DataType::Int64); - - match to_i64 { - Ok(series) => { - let nanosecs_per_day: i64 = 24 * 60 * 60 * 1_000_000_000; - let casted = series - .i64() - .map(|chunked| chunked.mul(nanosecs_per_day)) - .expect("already checked for casting"); - compare_casted_i64(Ok(&casted), val, f, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - DataType::Int64 => { - let casted = series.i64(); - compare_casted_i64(casted, val, f, span) - } - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: format!( - "Series of type {} can not be used for operations with an i64 value", - series.dtype() - ), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compare_casted_i64( - casted: Result<&ChunkedArray, PolarsError>, - val: i64, - f: F, - span: Span, -) -> Result -where - F: Fn(&ChunkedArray, i64) -> ChunkedArray, -{ - match casted { - Ok(casted) => { - let res = f(casted, val); - let res = res.into_series(); - NuDataFrame::series_to_value(res, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to i64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compare_series_float(series: &Series, val: f64, f: F, span: Span) -> Result -where - F: Fn(&ChunkedArray, f64) -> ChunkedArray, -{ - match series.dtype() { - DataType::Float32 => { - let to_f64 = series.cast(&DataType::Float64); - - match to_f64 { - Ok(series) => { - let casted = series.f64(); - compare_casted_f64(casted, val, f, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to i64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - DataType::Float64 => { - let casted = series.f64(); - compare_casted_f64(casted, val, f, span) - } - _ => Err(ShellError::GenericError { - error: "Incorrect type".into(), - msg: format!( - "Series of type {} can not be used for operations with a float value", - series.dtype() - ), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn compare_casted_f64( - casted: Result<&ChunkedArray, PolarsError>, - val: f64, - f: F, - span: Span, -) -> Result -where - F: Fn(&ChunkedArray, f64) -> ChunkedArray, -{ - match casted { - Ok(casted) => { - let res = f(casted, val); - let res = res.into_series(); - NuDataFrame::series_to_value(res, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to f64".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn contains_series_pat(series: &Series, pat: &str, span: Span) -> Result { - let casted = series.str(); - match casted { - Ok(casted) => { - let res = casted.contains(pat, false); - - match res { - Ok(res) => { - let res = res.into_series(); - NuDataFrame::series_to_value(res, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Error using contains".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to string".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn add_string_to_series(series: &Series, pat: &str, span: Span) -> Result { - let casted = series.str(); - match casted { - Ok(casted) => { - let res = casted + pat; - let res = res.into_series(); - - NuDataFrame::series_to_value(res, span) - } - Err(e) => Err(ShellError::GenericError { - error: "Unable to cast to string".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -#[cfg(test)] -mod test { - use super::*; - use nu_protocol::Span; - use polars::{prelude::NamedFrom, series::Series}; - - use crate::dataframe::values::NuDataFrame; - - #[test] - fn test_compute_between_series_comparisons() { - let series = Series::new("c", &[1, 2]); - let df = NuDataFrame::try_from_series(vec![series], Span::test_data()) - .expect("should be able to create a simple dataframe"); - - let c0 = df - .column("c", Span::test_data()) - .expect("should be able to get column c"); - - let c0_series = c0 - .as_series(Span::test_data()) - .expect("should be able to get series"); - - let c0_value = c0.into_value(Span::test_data()); - - let c1 = df - .column("c", Span::test_data()) - .expect("should be able to get column c"); - - let c1_series = c1 - .as_series(Span::test_data()) - .expect("should be able to get series"); - - let c1_value = c1.into_value(Span::test_data()); - - let op = Spanned { - item: Operator::Comparison(Comparison::NotEqual), - span: Span::test_data(), - }; - let result = compute_between_series(op, &c0_value, &c0_series, &c1_value, &c1_series) - .expect("compare should not fail"); - let result = NuDataFrame::try_from_value(result) - .expect("should be able to create a dataframe from a value"); - let result = result - .as_series(Span::test_data()) - .expect("should be convert to a series"); - assert_eq!(result, Series::new("neq_c_c", &[false, false])); - - let op = Spanned { - item: Operator::Comparison(Comparison::Equal), - span: Span::test_data(), - }; - let result = compute_between_series(op, &c0_value, &c0_series, &c1_value, &c1_series) - .expect("compare should not fail"); - let result = NuDataFrame::try_from_value(result) - .expect("should be able to create a dataframe from a value"); - let result = result - .as_series(Span::test_data()) - .expect("should be convert to a series"); - assert_eq!(result, Series::new("eq_c_c", &[true, true])); - - let op = Spanned { - item: Operator::Comparison(Comparison::LessThan), - span: Span::test_data(), - }; - let result = compute_between_series(op, &c0_value, &c0_series, &c1_value, &c1_series) - .expect("compare should not fail"); - let result = NuDataFrame::try_from_value(result) - .expect("should be able to create a dataframe from a value"); - let result = result - .as_series(Span::test_data()) - .expect("should be convert to a series"); - assert_eq!(result, Series::new("lt_c_c", &[false, false])); - - let op = Spanned { - item: Operator::Comparison(Comparison::LessThanOrEqual), - span: Span::test_data(), - }; - let result = compute_between_series(op, &c0_value, &c0_series, &c1_value, &c1_series) - .expect("compare should not fail"); - let result = NuDataFrame::try_from_value(result) - .expect("should be able to create a dataframe from a value"); - let result = result - .as_series(Span::test_data()) - .expect("should be convert to a series"); - assert_eq!(result, Series::new("lte_c_c", &[true, true])); - - let op = Spanned { - item: Operator::Comparison(Comparison::GreaterThan), - span: Span::test_data(), - }; - let result = compute_between_series(op, &c0_value, &c0_series, &c1_value, &c1_series) - .expect("compare should not fail"); - let result = NuDataFrame::try_from_value(result) - .expect("should be able to create a dataframe from a value"); - let result = result - .as_series(Span::test_data()) - .expect("should be convert to a series"); - assert_eq!(result, Series::new("gt_c_c", &[false, false])); - - let op = Spanned { - item: Operator::Comparison(Comparison::GreaterThanOrEqual), - span: Span::test_data(), - }; - let result = compute_between_series(op, &c0_value, &c0_series, &c1_value, &c1_series) - .expect("compare should not fail"); - let result = NuDataFrame::try_from_value(result) - .expect("should be able to create a dataframe from a value"); - let result = result - .as_series(Span::test_data()) - .expect("should be convert to a series"); - assert_eq!(result, Series::new("gte_c_c", &[true, true])); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/conversion.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/conversion.rs deleted file mode 100644 index 7ab339d78d..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/conversion.rs +++ /dev/null @@ -1,1435 +0,0 @@ -use super::{DataFrameValue, NuDataFrame, NuSchema}; -use chrono::{DateTime, Duration, FixedOffset, NaiveTime, TimeZone, Utc}; -use chrono_tz::Tz; -use indexmap::map::{Entry, IndexMap}; -use nu_protocol::{Record, ShellError, Span, Value}; -use polars::{ - chunked_array::{ - builder::AnonymousOwnedListBuilder, object::builder::ObjectChunkedBuilder, ChunkedArray, - }, - datatypes::AnyValue, - export::arrow::Either, - prelude::{ - DataFrame, DataType, DatetimeChunked, Float32Type, Float64Type, Int16Type, Int32Type, - Int64Type, Int8Type, IntoSeries, ListBooleanChunkedBuilder, ListBuilderTrait, - ListPrimitiveChunkedBuilder, ListStringChunkedBuilder, ListType, NamedFrom, - NewChunkedArray, ObjectType, Schema, Series, StructChunked, TemporalMethods, TimeUnit, - UInt16Type, UInt32Type, UInt64Type, UInt8Type, - }, -}; -use std::ops::{Deref, DerefMut}; - -const NANOS_PER_DAY: i64 = 86_400_000_000_000; - -// The values capacity is for the size of an vec. -// Since this is impossible to determine without traversing every value -// I just picked one. Since this is for converting back and forth -// between nushell tables the values shouldn't be too extremely large for -// practical reasons (~ a few thousand rows). -const VALUES_CAPACITY: usize = 10; - -macro_rules! value_to_primitive { - ($value:ident, u8) => { - $value.as_i64().map(|v| v as u8) - }; - ($value:ident, u16) => { - $value.as_i64().map(|v| v as u16) - }; - ($value:ident, u32) => { - $value.as_i64().map(|v| v as u32) - }; - ($value:ident, u64) => { - $value.as_i64().map(|v| v as u64) - }; - ($value:ident, i8) => { - $value.as_i64().map(|v| v as i8) - }; - ($value:ident, i16) => { - $value.as_i64().map(|v| v as i16) - }; - ($value:ident, i32) => { - $value.as_i64().map(|v| v as i32) - }; - ($value:ident, i64) => { - $value.as_i64() - }; - ($value:ident, f32) => { - $value.as_f64().map(|v| v as f32) - }; - ($value:ident, f64) => { - $value.as_f64() - }; -} - -#[derive(Debug)] -pub struct Column { - name: String, - values: Vec, -} - -impl Column { - pub fn new(name: String, values: Vec) -> Self { - Self { name, values } - } - - pub fn new_empty(name: String) -> Self { - Self { - name, - values: Vec::new(), - } - } - - pub fn name(&self) -> &str { - self.name.as_str() - } -} - -impl IntoIterator for Column { - type Item = Value; - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.values.into_iter() - } -} - -impl Deref for Column { - type Target = Vec; - - fn deref(&self) -> &Self::Target { - &self.values - } -} - -impl DerefMut for Column { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.values - } -} - -#[derive(Debug)] -pub struct TypedColumn { - column: Column, - column_type: Option, -} - -impl TypedColumn { - fn new_empty(name: String) -> Self { - Self { - column: Column::new_empty(name), - column_type: None, - } - } -} - -impl Deref for TypedColumn { - type Target = Column; - - fn deref(&self) -> &Self::Target { - &self.column - } -} - -impl DerefMut for TypedColumn { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.column - } -} - -pub type ColumnMap = IndexMap; - -pub fn create_column( - series: &Series, - from_row: usize, - to_row: usize, - span: Span, -) -> Result { - let size = to_row - from_row; - let values = series_to_values(series, Some(from_row), Some(size), span)?; - Ok(Column::new(series.name().into(), values)) -} - -// Adds a separator to the vector of values using the column names from the -// dataframe to create the Values Row -pub fn add_separator(values: &mut Vec, df: &DataFrame, span: Span) { - let mut record = Record::new(); - - record.push("index", Value::string("...", span)); - - for name in df.get_column_names() { - record.push(name, Value::string("...", span)) - } - - values.push(Value::record(record, span)); -} - -// Inserting the values found in a Value::List or Value::Record -pub fn insert_record( - column_values: &mut ColumnMap, - record: Record, - maybe_schema: &Option, -) -> Result<(), ShellError> { - for (col, value) in record { - insert_value(value, col, column_values, maybe_schema)?; - } - - Ok(()) -} - -pub fn insert_value( - value: Value, - key: String, - column_values: &mut ColumnMap, - maybe_schema: &Option, -) -> Result<(), ShellError> { - let col_val = match column_values.entry(key.clone()) { - Entry::Vacant(entry) => entry.insert(TypedColumn::new_empty(key.clone())), - Entry::Occupied(entry) => entry.into_mut(), - }; - - // Checking that the type for the value is the same - // for the previous value in the column - if col_val.values.is_empty() { - if let Some(schema) = maybe_schema { - if let Some(field) = schema.schema.get_field(&key) { - col_val.column_type = Some(field.data_type().clone()); - } - } - - if col_val.column_type.is_none() { - col_val.column_type = Some(value_to_data_type(&value)); - } - - col_val.values.push(value); - } else { - let prev_value = &col_val.values[col_val.values.len() - 1]; - - match (&prev_value, &value) { - (Value::Int { .. }, Value::Int { .. }) - | (Value::Float { .. }, Value::Float { .. }) - | (Value::String { .. }, Value::String { .. }) - | (Value::Bool { .. }, Value::Bool { .. }) - | (Value::Date { .. }, Value::Date { .. }) - | (Value::Filesize { .. }, Value::Filesize { .. }) - | (Value::Duration { .. }, Value::Duration { .. }) => col_val.values.push(value), - (Value::List { .. }, _) => { - col_val.column_type = Some(value_to_data_type(&value)); - col_val.values.push(value); - } - _ => { - col_val.column_type = Some(DataType::Object("Value", None)); - col_val.values.push(value); - } - } - } - - Ok(()) -} - -fn value_to_data_type(value: &Value) -> DataType { - match &value { - Value::Int { .. } => DataType::Int64, - Value::Float { .. } => DataType::Float64, - Value::String { .. } => DataType::String, - Value::Bool { .. } => DataType::Boolean, - Value::Date { .. } => DataType::Date, - Value::Duration { .. } => DataType::Duration(TimeUnit::Nanoseconds), - Value::Filesize { .. } => DataType::Int64, - Value::List { vals, .. } => { - // We need to determined the type inside of the list. - // Since Value::List does not have any kind of - // type information, we need to look inside the list. - // This will cause errors if lists have inconsistent types. - // Basically, if a list column needs to be converted to dataframe, - // needs to have consistent types. - let list_type = vals - .iter() - .filter(|v| !matches!(v, Value::Nothing { .. })) - .map(value_to_data_type) - .nth(1) - .unwrap_or(DataType::Object("Value", None)); - - DataType::List(Box::new(list_type)) - } - _ => DataType::Object("Value", None), - } -} - -fn typed_column_to_series(name: &str, column: TypedColumn) -> Result { - if let Some(column_type) = &column.column_type { - match column_type { - DataType::Float32 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_f64().map(|v| v as f32)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Float64 => { - let series_values: Result, _> = - column.values.iter().map(|v| v.as_f64()).collect(); - Ok(Series::new(name, series_values?)) - } - DataType::UInt8 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as u8)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::UInt16 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as u16)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::UInt32 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as u32)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::UInt64 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as u64)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Int8 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as i8)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Int16 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as i16)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Int32 => { - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| v as i32)) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Int64 => { - let series_values: Result, _> = - column.values.iter().map(|v| v.as_i64()).collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Boolean => { - let series_values: Result, _> = - column.values.iter().map(|v| v.as_bool()).collect(); - Ok(Series::new(name, series_values?)) - } - DataType::String => { - let series_values: Result, _> = - column.values.iter().map(|v| v.coerce_string()).collect(); - Ok(Series::new(name, series_values?)) - } - DataType::Object(_, _) => value_to_series(name, &column.values), - DataType::Duration(time_unit) => { - //todo - finish type conversion - let series_values: Result, _> = column - .values - .iter() - .map(|v| v.as_i64().map(|v| nanos_from_timeunit(v, *time_unit))) - .collect(); - Ok(Series::new(name, series_values?)) - } - DataType::List(list_type) => { - match input_type_list_to_series(name, list_type.as_ref(), &column.values) { - Ok(series) => Ok(series), - Err(_) => { - // An error case will occur when there are lists of mixed types. - // If this happens, fallback to object list - input_type_list_to_series( - name, - &DataType::Object("unknown", None), - &column.values, - ) - } - } - } - DataType::Date => { - let it = column.values.iter().map(|v| { - if let Value::Date { val, .. } = &v { - Some(val.timestamp_nanos_opt().unwrap_or_default()) - } else { - None - } - }); - - let res: DatetimeChunked = ChunkedArray::::from_iter_options(name, it) - .into_datetime(TimeUnit::Nanoseconds, None); - - Ok(res.into_series()) - } - DataType::Datetime(tu, maybe_tz) => { - let dates = column - .values - .iter() - .map(|v| { - if let Value::Date { val, .. } = &v { - // If there is a timezone specified, make sure - // the value is converted to it - Ok(maybe_tz - .as_ref() - .map(|tz| tz.parse::().map(|tz| val.with_timezone(&tz))) - .transpose() - .map_err(|e| ShellError::GenericError { - error: "Error parsing timezone".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })? - .and_then(|dt| dt.timestamp_nanos_opt()) - .map(|nanos| nanos_from_timeunit(nanos, *tu))) - } else { - Ok(None) - } - }) - .collect::>, ShellError>>()?; - - let res: DatetimeChunked = - ChunkedArray::::from_iter_options(name, dates.into_iter()) - .into_datetime(*tu, maybe_tz.clone()); - - Ok(res.into_series()) - } - DataType::Struct(fields) => { - let schema = Some(NuSchema::new(Schema::from_iter(fields.clone()))); - let mut structs: Vec = Vec::new(); - - for v in column.values.iter() { - let mut column_values: ColumnMap = IndexMap::new(); - let record = v.as_record()?; - insert_record(&mut column_values, record.clone(), &schema)?; - let df = from_parsed_columns(column_values)?; - structs.push(df.as_series(Span::unknown())?); - } - - let chunked = StructChunked::new(column.name(), structs.as_ref()).map_err(|e| { - ShellError::GenericError { - error: format!("Error creating struct: {e}"), - msg: "".into(), - span: None, - help: None, - inner: vec![], - } - })?; - Ok(chunked.into_series()) - } - _ => Err(ShellError::GenericError { - error: format!("Error creating dataframe: Unsupported type: {column_type:?}"), - msg: "".into(), - span: None, - help: None, - inner: vec![], - }), - } - } else { - Err(ShellError::GenericError { - error: "Passed a type column with no type".into(), - msg: "".into(), - span: None, - help: None, - inner: vec![], - }) - } -} - -// The ColumnMap has the parsed data from the StreamInput -// This data can be used to create a Series object that can initialize -// the dataframe based on the type of data that is found -pub fn from_parsed_columns(column_values: ColumnMap) -> Result { - let mut df_series: Vec = Vec::new(); - for (name, column) in column_values { - let series = typed_column_to_series(&name, column)?; - df_series.push(series); - } - - DataFrame::new(df_series) - .map(|df| NuDataFrame::new(false, df)) - .map_err(|e| ShellError::GenericError { - error: "Error creating dataframe".into(), - msg: e.to_string(), - span: None, - help: None, - inner: vec![], - }) -} - -fn value_to_series(name: &str, values: &[Value]) -> Result { - let mut builder = ObjectChunkedBuilder::::new(name, values.len()); - - for v in values { - builder.append_value(DataFrameValue::new(v.clone())); - } - - let res = builder.finish(); - Ok(res.into_series()) -} - -fn input_type_list_to_series( - name: &str, - data_type: &DataType, - values: &[Value], -) -> Result { - let inconsistent_error = |_| ShellError::GenericError { - error: format!( - "column {name} contains a list with inconsistent types: Expecting: {data_type:?}" - ), - msg: "".into(), - span: None, - help: None, - inner: vec![], - }; - - macro_rules! primitive_list_series { - ($list_type:ty, $vec_type:tt) => {{ - let mut builder = ListPrimitiveChunkedBuilder::<$list_type>::new( - name, - values.len(), - VALUES_CAPACITY, - data_type.clone(), - ); - - for v in values { - let value_list = v - .as_list()? - .iter() - .map(|v| value_to_primitive!(v, $vec_type)) - .collect::, _>>() - .map_err(inconsistent_error)?; - builder.append_iter_values(value_list.iter().copied()); - } - let res = builder.finish(); - Ok(res.into_series()) - }}; - } - - match *data_type { - // list of boolean values - DataType::Boolean => { - let mut builder = ListBooleanChunkedBuilder::new(name, values.len(), VALUES_CAPACITY); - for v in values { - let value_list = v - .as_list()? - .iter() - .map(|v| v.as_bool()) - .collect::, _>>() - .map_err(inconsistent_error)?; - builder.append_iter(value_list.iter().map(|v| Some(*v))); - } - let res = builder.finish(); - Ok(res.into_series()) - } - DataType::Duration(_) => primitive_list_series!(Int64Type, i64), - DataType::UInt8 => primitive_list_series!(UInt8Type, u8), - DataType::UInt16 => primitive_list_series!(UInt16Type, u16), - DataType::UInt32 => primitive_list_series!(UInt32Type, u32), - DataType::UInt64 => primitive_list_series!(UInt64Type, u64), - DataType::Int8 => primitive_list_series!(Int8Type, i8), - DataType::Int16 => primitive_list_series!(Int16Type, i16), - DataType::Int32 => primitive_list_series!(Int32Type, i32), - DataType::Int64 => primitive_list_series!(Int64Type, i64), - DataType::Float32 => primitive_list_series!(Float32Type, f32), - DataType::Float64 => primitive_list_series!(Float64Type, f64), - DataType::String => { - let mut builder = ListStringChunkedBuilder::new(name, values.len(), VALUES_CAPACITY); - for v in values { - let value_list = v - .as_list()? - .iter() - .map(|v| v.coerce_string()) - .collect::, _>>() - .map_err(inconsistent_error)?; - builder.append_values_iter(value_list.iter().map(AsRef::as_ref)); - } - let res = builder.finish(); - Ok(res.into_series()) - } - DataType::Date => { - let mut builder = AnonymousOwnedListBuilder::new( - name, - values.len(), - Some(DataType::Datetime(TimeUnit::Nanoseconds, None)), - ); - for (i, v) in values.iter().enumerate() { - let list_name = i.to_string(); - - let it = v.as_list()?.iter().map(|v| { - if let Value::Date { val, .. } = &v { - Some(val.timestamp_nanos_opt().unwrap_or_default()) - } else { - None - } - }); - let dt_chunked = ChunkedArray::::from_iter_options(&list_name, it) - .into_datetime(TimeUnit::Nanoseconds, None); - - builder - .append_series(&dt_chunked.into_series()) - .map_err(|e| ShellError::GenericError { - error: "Error appending to series".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })? - } - let res = builder.finish(); - Ok(res.into_series()) - } - DataType::List(ref sub_list_type) => { - Ok(input_type_list_to_series(name, sub_list_type, values)?) - } - // treat everything else as an object - _ => Ok(value_to_series(name, values)?), - } -} - -fn series_to_values( - series: &Series, - maybe_from_row: Option, - maybe_size: Option, - span: Span, -) -> Result, ShellError> { - match series.dtype() { - DataType::Null => { - let it = std::iter::repeat(Value::nothing(span)); - let values = if let Some(size) = maybe_size { - Either::Left(it.take(size)) - } else { - Either::Right(it) - } - .collect::>(); - - Ok(values) - } - DataType::UInt8 => { - let casted = series.u8().map_err(|e| ShellError::GenericError { - error: "Error casting column to u8".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::UInt16 => { - let casted = series.u16().map_err(|e| ShellError::GenericError { - error: "Error casting column to u16".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::UInt32 => { - let casted = series.u32().map_err(|e| ShellError::GenericError { - error: "Error casting column to u32".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::UInt64 => { - let casted = series.u64().map_err(|e| ShellError::GenericError { - error: "Error casting column to u64".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Int8 => { - let casted = series.i8().map_err(|e| ShellError::GenericError { - error: "Error casting column to i8".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Int16 => { - let casted = series.i16().map_err(|e| ShellError::GenericError { - error: "Error casting column to i16".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Int32 => { - let casted = series.i32().map_err(|e| ShellError::GenericError { - error: "Error casting column to i32".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a as i64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Int64 => { - let casted = series.i64().map_err(|e| ShellError::GenericError { - error: "Error casting column to i64".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::int(a, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Float32 => { - let casted = series.f32().map_err(|e| ShellError::GenericError { - error: "Error casting column to f32".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::float(a as f64, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Float64 => { - let casted = series.f64().map_err(|e| ShellError::GenericError { - error: "Error casting column to f64".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::float(a, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Boolean => { - let casted = series.bool().map_err(|e| ShellError::GenericError { - error: "Error casting column to bool".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::bool(a, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::String => { - let casted = series.str().map_err(|e| ShellError::GenericError { - error: "Error casting column to string".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => Value::string(a.to_string(), span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - DataType::Object(x, _) => { - let casted = series - .as_any() - .downcast_ref::>>(); - - match casted { - None => Err(ShellError::GenericError { - error: "Error casting object from series".into(), - msg: "".into(), - span: None, - help: Some(format!("Object not supported for conversion: {x}")), - inner: vec![], - }), - Some(ca) => { - let it = ca.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) - { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => a.get_value(), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - } - } - DataType::List(x) => { - let casted = series.as_any().downcast_ref::>(); - match casted { - None => Err(ShellError::GenericError { - error: "Error casting list from series".into(), - msg: "".into(), - span: None, - help: Some(format!("List not supported for conversion: {x}")), - inner: vec![], - }), - Some(ca) => { - let it = ca.into_iter(); - if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|ca| { - let sublist: Vec = if let Some(ref s) = ca { - series_to_values(s, None, None, Span::unknown())? - } else { - // empty item - vec![] - }; - Ok(Value::list(sublist, span)) - }) - .collect::, ShellError>>() - } - } - } - DataType::Date => { - let casted = series.date().map_err(|e| ShellError::GenericError { - error: "Error casting column to date".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => { - let nanos = nanos_per_day(a); - let datetime = datetime_from_epoch_nanos(nanos, &None, span)?; - Ok(Value::date(datetime, span)) - } - None => Ok(Value::nothing(span)), - }) - .collect::, ShellError>>()?; - Ok(values) - } - DataType::Datetime(time_unit, tz) => { - let casted = series.datetime().map_err(|e| ShellError::GenericError { - error: "Error casting column to datetime".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(a) => { - // elapsed time in nano/micro/milliseconds since 1970-01-01 - let nanos = nanos_from_timeunit(a, *time_unit); - let datetime = datetime_from_epoch_nanos(nanos, tz, span)?; - Ok(Value::date(datetime, span)) - } - None => Ok(Value::nothing(span)), - }) - .collect::, ShellError>>()?; - Ok(values) - } - DataType::Struct(polar_fields) => { - let casted = series.struct_().map_err(|e| ShellError::GenericError { - error: "Error casting column to struct".into(), - msg: "".to_string(), - span: None, - help: Some(e.to_string()), - inner: Vec::new(), - })?; - let it = casted.into_iter(); - let values: Result, ShellError> = - if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|any_values| { - let record = polar_fields - .iter() - .zip(any_values) - .map(|(field, val)| { - any_value_to_value(val, span).map(|val| (field.name.to_string(), val)) - }) - .collect::>()?; - - Ok(Value::record(record, span)) - }) - .collect(); - values - } - DataType::Time => { - let casted = - series - .timestamp(TimeUnit::Nanoseconds) - .map_err(|e| ShellError::GenericError { - error: "Error casting column to time".into(), - msg: "".into(), - span: None, - help: Some(e.to_string()), - inner: vec![], - })?; - - let it = casted.into_iter(); - let values = if let (Some(size), Some(from_row)) = (maybe_size, maybe_from_row) { - Either::Left(it.skip(from_row).take(size)) - } else { - Either::Right(it) - } - .map(|v| match v { - Some(nanoseconds) => Value::duration(nanoseconds, span), - None => Value::nothing(span), - }) - .collect::>(); - - Ok(values) - } - e => Err(ShellError::GenericError { - error: "Error creating Dataframe".into(), - msg: "".to_string(), - span: None, - help: Some(format!("Value not supported in nushell: {e}")), - inner: vec![], - }), - } -} - -fn any_value_to_value(any_value: &AnyValue, span: Span) -> Result { - match any_value { - AnyValue::Null => Ok(Value::nothing(span)), - AnyValue::Boolean(b) => Ok(Value::bool(*b, span)), - AnyValue::String(s) => Ok(Value::string(s.to_string(), span)), - AnyValue::UInt8(i) => Ok(Value::int(*i as i64, span)), - AnyValue::UInt16(i) => Ok(Value::int(*i as i64, span)), - AnyValue::UInt32(i) => Ok(Value::int(*i as i64, span)), - AnyValue::UInt64(i) => Ok(Value::int(*i as i64, span)), - AnyValue::Int8(i) => Ok(Value::int(*i as i64, span)), - AnyValue::Int16(i) => Ok(Value::int(*i as i64, span)), - AnyValue::Int32(i) => Ok(Value::int(*i as i64, span)), - AnyValue::Int64(i) => Ok(Value::int(*i, span)), - AnyValue::Float32(f) => Ok(Value::float(*f as f64, span)), - AnyValue::Float64(f) => Ok(Value::float(*f, span)), - AnyValue::Date(d) => { - let nanos = nanos_per_day(*d); - datetime_from_epoch_nanos(nanos, &None, span) - .map(|datetime| Value::date(datetime, span)) - } - AnyValue::Datetime(a, time_unit, tz) => { - let nanos = nanos_from_timeunit(*a, *time_unit); - datetime_from_epoch_nanos(nanos, tz, span).map(|datetime| Value::date(datetime, span)) - } - AnyValue::Duration(a, time_unit) => { - let nanos = match time_unit { - TimeUnit::Nanoseconds => *a, - TimeUnit::Microseconds => *a * 1_000, - TimeUnit::Milliseconds => *a * 1_000_000, - }; - Ok(Value::duration(nanos, span)) - } - // AnyValue::Time represents the current time since midnight. - // Unfortunately, there is no timezone related information. - // Given this, calculate the current date from UTC and add the time. - AnyValue::Time(nanos) => time_from_midnight(*nanos, span), - AnyValue::List(series) => { - series_to_values(series, None, None, span).map(|values| Value::list(values, span)) - } - AnyValue::Struct(_idx, _struct_array, _s_fields) => { - // This should convert to a StructOwned object. - let static_value = - any_value - .clone() - .into_static() - .map_err(|e| ShellError::GenericError { - error: "Cannot convert polars struct to static value".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: Vec::new(), - })?; - any_value_to_value(&static_value, span) - } - AnyValue::StructOwned(struct_tuple) => { - let record = struct_tuple - .1 - .iter() - .zip(&struct_tuple.0) - .map(|(field, val)| { - any_value_to_value(val, span).map(|val| (field.name.to_string(), val)) - }) - .collect::>()?; - - Ok(Value::record(record, span)) - } - AnyValue::StringOwned(s) => Ok(Value::string(s.to_string(), span)), - AnyValue::Binary(bytes) => Ok(Value::binary(*bytes, span)), - AnyValue::BinaryOwned(bytes) => Ok(Value::binary(bytes.to_owned(), span)), - e => Err(ShellError::GenericError { - error: "Error creating Value".into(), - msg: "".to_string(), - span: None, - help: Some(format!("Value not supported in nushell: {e}")), - inner: Vec::new(), - }), - } -} - -fn nanos_per_day(days: i32) -> i64 { - days as i64 * NANOS_PER_DAY -} - -fn nanos_from_timeunit(a: i64, time_unit: TimeUnit) -> i64 { - a * match time_unit { - TimeUnit::Microseconds => 1_000, // Convert microseconds to nanoseconds - TimeUnit::Milliseconds => 1_000_000, // Convert milliseconds to nanoseconds - TimeUnit::Nanoseconds => 1, // Already in nanoseconds - } -} - -fn datetime_from_epoch_nanos( - nanos: i64, - timezone: &Option, - span: Span, -) -> Result, ShellError> { - let tz: Tz = if let Some(polars_tz) = timezone { - polars_tz - .parse::() - .map_err(|_| ShellError::GenericError { - error: format!("Could not parse polars timezone: {polars_tz}"), - msg: "".to_string(), - span: Some(span), - help: None, - inner: vec![], - })? - } else { - Tz::UTC - }; - - Ok(tz.timestamp_nanos(nanos).fixed_offset()) -} - -fn time_from_midnight(nanos: i64, span: Span) -> Result { - let today = Utc::now().date_naive(); - NaiveTime::from_hms_opt(0, 0, 0) // midnight - .map(|time| time + Duration::nanoseconds(nanos)) // current time - .map(|time| today.and_time(time)) // current date and time - .and_then(|datetime| { - FixedOffset::east_opt(0) // utc - .map(|offset| { - DateTime::::from_naive_utc_and_offset(datetime, offset) - }) - }) - .map(|datetime| Value::date(datetime, span)) // current date and time - .ok_or(ShellError::CantConvert { - to_type: "datetime".to_string(), - from_type: "polars time".to_string(), - span, - help: Some("Could not convert polars time of {nanos} to datetime".to_string()), - }) -} - -#[cfg(test)] -mod tests { - use indexmap::indexmap; - use nu_protocol::record; - use polars::export::arrow::array::{BooleanArray, PrimitiveArray}; - use polars::prelude::Field; - use polars_io::prelude::StructArray; - - use super::*; - - #[test] - fn test_parsed_column_string_list() -> Result<(), Box> { - let values = vec![ - Value::list( - vec![Value::string("bar".to_string(), Span::test_data())], - Span::test_data(), - ), - Value::list( - vec![Value::string("baz".to_string(), Span::test_data())], - Span::test_data(), - ), - ]; - let column = Column { - name: "foo".to_string(), - values: values.clone(), - }; - let typed_column = TypedColumn { - column, - column_type: Some(DataType::List(Box::new(DataType::String))), - }; - - let column_map = indexmap!("foo".to_string() => typed_column); - let parsed_df = from_parsed_columns(column_map)?; - let parsed_columns = parsed_df.columns(Span::test_data())?; - assert_eq!(parsed_columns.len(), 1); - let column = parsed_columns - .first() - .expect("There should be a first value in columns"); - assert_eq!(column.name(), "foo"); - assert_eq!(column.values, values); - - Ok(()) - } - - #[test] - fn test_any_value_to_value() -> Result<(), Box> { - let span = Span::test_data(); - assert_eq!( - any_value_to_value(&AnyValue::Null, span)?, - Value::nothing(span) - ); - - let test_bool = true; - assert_eq!( - any_value_to_value(&AnyValue::Boolean(test_bool), span)?, - Value::bool(test_bool, span) - ); - - let test_str = "foo"; - assert_eq!( - any_value_to_value(&AnyValue::String(test_str), span)?, - Value::string(test_str.to_string(), span) - ); - assert_eq!( - any_value_to_value(&AnyValue::StringOwned(test_str.into()), span)?, - Value::string(test_str.to_owned(), span) - ); - - let tests_uint8 = 4; - assert_eq!( - any_value_to_value(&AnyValue::UInt8(tests_uint8), span)?, - Value::int(tests_uint8 as i64, span) - ); - - let tests_uint16 = 233; - assert_eq!( - any_value_to_value(&AnyValue::UInt16(tests_uint16), span)?, - Value::int(tests_uint16 as i64, span) - ); - - let tests_uint32 = 897688233; - assert_eq!( - any_value_to_value(&AnyValue::UInt32(tests_uint32), span)?, - Value::int(tests_uint32 as i64, span) - ); - - let tests_uint64 = 903225135897388233; - assert_eq!( - any_value_to_value(&AnyValue::UInt64(tests_uint64), span)?, - Value::int(tests_uint64 as i64, span) - ); - - let tests_float32 = 903225135897388233.3223353; - assert_eq!( - any_value_to_value(&AnyValue::Float32(tests_float32), span)?, - Value::float(tests_float32 as f64, span) - ); - - let tests_float64 = 9064251358973882322333.64233533232; - assert_eq!( - any_value_to_value(&AnyValue::Float64(tests_float64), span)?, - Value::float(tests_float64, span) - ); - - let test_days = 10_957; - let comparison_date = Utc - .with_ymd_and_hms(2000, 1, 1, 0, 0, 0) - .unwrap() - .fixed_offset(); - assert_eq!( - any_value_to_value(&AnyValue::Date(test_days), span)?, - Value::date(comparison_date, span) - ); - - let test_millis = 946_684_800_000; - assert_eq!( - any_value_to_value( - &AnyValue::Datetime(test_millis, TimeUnit::Milliseconds, &None), - span - )?, - Value::date(comparison_date, span) - ); - - let test_duration_millis = 99_999; - let test_duration_micros = 99_999_000; - let test_duration_nanos = 99_999_000_000; - assert_eq!( - any_value_to_value( - &AnyValue::Duration(test_duration_nanos, TimeUnit::Nanoseconds), - span - )?, - Value::duration(test_duration_nanos, span) - ); - assert_eq!( - any_value_to_value( - &AnyValue::Duration(test_duration_micros, TimeUnit::Microseconds), - span - )?, - Value::duration(test_duration_nanos, span) - ); - assert_eq!( - any_value_to_value( - &AnyValue::Duration(test_duration_millis, TimeUnit::Milliseconds), - span - )?, - Value::duration(test_duration_nanos, span) - ); - - let test_binary = b"sdf2332f32q3f3afwaf3232f32"; - assert_eq!( - any_value_to_value(&AnyValue::Binary(test_binary), span)?, - Value::binary(test_binary.to_vec(), span) - ); - assert_eq!( - any_value_to_value(&AnyValue::BinaryOwned(test_binary.to_vec()), span)?, - Value::binary(test_binary.to_vec(), span) - ); - - let test_time_nanos = 54_000_000_000_000; - let test_time = DateTime::::from_naive_utc_and_offset( - Utc::now() - .date_naive() - .and_time(NaiveTime::from_hms_opt(15, 00, 00).unwrap()), - FixedOffset::east_opt(0).unwrap(), - ); - assert_eq!( - any_value_to_value(&AnyValue::Time(test_time_nanos), span)?, - Value::date(test_time, span) - ); - - let test_list_series = Series::new("int series", &[1, 2, 3]); - let comparison_list_series = Value::list( - vec![ - Value::int(1, span), - Value::int(2, span), - Value::int(3, span), - ], - span, - ); - assert_eq!( - any_value_to_value(&AnyValue::List(test_list_series), span)?, - comparison_list_series - ); - - let field_value_0 = AnyValue::Int32(1); - let field_value_1 = AnyValue::Boolean(true); - let values = vec![field_value_0, field_value_1]; - let field_name_0 = "num_field"; - let field_name_1 = "bool_field"; - let fields = vec![ - Field::new(field_name_0, DataType::Int32), - Field::new(field_name_1, DataType::Boolean), - ]; - let test_owned_struct = AnyValue::StructOwned(Box::new((values, fields.clone()))); - let comparison_owned_record = Value::test_record(record!( - field_name_0 => Value::int(1, span), - field_name_1 => Value::bool(true, span), - )); - assert_eq!( - any_value_to_value(&test_owned_struct, span)?, - comparison_owned_record.clone() - ); - - let test_int_arr = PrimitiveArray::from([Some(1_i32)]); - let test_bool_arr = BooleanArray::from([Some(true)]); - let test_struct_arr = StructArray::new( - DataType::Struct(fields.clone()).to_arrow(true), - vec![Box::new(test_int_arr), Box::new(test_bool_arr)], - None, - ); - assert_eq!( - any_value_to_value( - &AnyValue::Struct(0, &test_struct_arr, fields.as_slice()), - span - )?, - comparison_owned_record - ); - - Ok(()) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/custom_value.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/custom_value.rs deleted file mode 100644 index da8b27398b..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/custom_value.rs +++ /dev/null @@ -1,79 +0,0 @@ -use super::NuDataFrame; -use nu_protocol::{ast::Operator, CustomValue, ShellError, Span, Value}; - -// CustomValue implementation for NuDataFrame -impl CustomValue for NuDataFrame { - fn typetag_name(&self) -> &'static str { - "dataframe" - } - - fn typetag_deserialize(&self) { - unimplemented!("typetag_deserialize") - } - - fn clone_value(&self, span: nu_protocol::Span) -> Value { - let cloned = NuDataFrame { - df: self.df.clone(), - from_lazy: false, - }; - - Value::custom(Box::new(cloned), span) - } - - fn type_name(&self) -> String { - self.typetag_name().to_string() - } - - fn to_base_value(&self, span: Span) -> Result { - let vals = self.print(span)?; - - Ok(Value::list(vals, span)) - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - fn follow_path_int( - &self, - _self_span: Span, - count: usize, - path_span: Span, - ) -> Result { - self.get_value(count, path_span) - } - - fn follow_path_string( - &self, - _self_span: Span, - column_name: String, - path_span: Span, - ) -> Result { - let column = self.column(&column_name, path_span)?; - Ok(column.into_value(path_span)) - } - - fn partial_cmp(&self, other: &Value) -> Option { - match other { - Value::Custom { val, .. } => val - .as_any() - .downcast_ref::() - .and_then(|other| self.is_equal(other)), - _ => None, - } - } - - fn operation( - &self, - lhs_span: Span, - operator: Operator, - op: Span, - right: &Value, - ) -> Result { - self.compute_with_value(lhs_span, operator, op, right) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs deleted file mode 100644 index 967e03580f..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/mod.rs +++ /dev/null @@ -1,580 +0,0 @@ -mod between_values; -mod conversion; -mod custom_value; -mod operations; - -pub use conversion::{Column, ColumnMap}; -pub use operations::Axis; - -use super::{nu_schema::NuSchema, utils::DEFAULT_ROWS, NuLazyFrame}; -use indexmap::IndexMap; -use nu_protocol::{did_you_mean, PipelineData, Record, ShellError, Span, Value}; -use polars::{ - chunked_array::ops::SortMultipleOptions, - prelude::{DataFrame, DataType, IntoLazy, LazyFrame, PolarsObject, Series}, -}; -use polars_plan::prelude::{lit, Expr, Null}; -use polars_utils::total_ord::{TotalEq, TotalHash}; -use serde::{Deserialize, Serialize}; -use std::{ - cmp::Ordering, - collections::HashSet, - fmt::Display, - hash::{Hash, Hasher}, -}; - -// DataFrameValue is an encapsulation of Nushell Value that can be used -// to define the PolarsObject Trait. The polars object trait allows to -// create dataframes with mixed datatypes -#[derive(Clone, Debug)] -pub struct DataFrameValue(Value); - -impl DataFrameValue { - fn new(value: Value) -> Self { - Self(value) - } - - fn get_value(&self) -> Value { - self.0.clone() - } -} - -impl TotalHash for DataFrameValue { - fn tot_hash(&self, state: &mut H) - where - H: Hasher, - { - (*self).hash(state) - } -} - -impl Display for DataFrameValue { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0.get_type()) - } -} - -impl Default for DataFrameValue { - fn default() -> Self { - Self(Value::nothing(Span::unknown())) - } -} - -impl PartialEq for DataFrameValue { - fn eq(&self, other: &Self) -> bool { - self.0.partial_cmp(&other.0).map_or(false, Ordering::is_eq) - } -} -impl Eq for DataFrameValue {} - -impl Hash for DataFrameValue { - fn hash(&self, state: &mut H) { - match &self.0 { - Value::Nothing { .. } => 0.hash(state), - Value::Int { val, .. } => val.hash(state), - Value::String { val, .. } => val.hash(state), - // TODO. Define hash for the rest of types - _ => {} - } - } -} - -impl TotalEq for DataFrameValue { - fn tot_eq(&self, other: &Self) -> bool { - self == other - } -} - -impl PolarsObject for DataFrameValue { - fn type_name() -> &'static str { - "object" - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct NuDataFrame { - pub df: DataFrame, - pub from_lazy: bool, -} - -impl AsRef for NuDataFrame { - fn as_ref(&self) -> &polars::prelude::DataFrame { - &self.df - } -} - -impl AsMut for NuDataFrame { - fn as_mut(&mut self) -> &mut polars::prelude::DataFrame { - &mut self.df - } -} - -impl From for NuDataFrame { - fn from(df: DataFrame) -> Self { - Self { - df, - from_lazy: false, - } - } -} - -impl NuDataFrame { - pub fn new(from_lazy: bool, df: DataFrame) -> Self { - Self { df, from_lazy } - } - - pub fn lazy(&self) -> LazyFrame { - self.df.clone().lazy() - } - - fn default_value(span: Span) -> Value { - let dataframe = DataFrame::default(); - NuDataFrame::dataframe_into_value(dataframe, span) - } - - pub fn dataframe_into_value(dataframe: DataFrame, span: Span) -> Value { - Value::custom(Box::new(Self::new(false, dataframe)), span) - } - - pub fn into_value(self, span: Span) -> Value { - if self.from_lazy { - let lazy = NuLazyFrame::from_dataframe(self); - Value::custom(Box::new(lazy), span) - } else { - Value::custom(Box::new(self), span) - } - } - - pub fn series_to_value(series: Series, span: Span) -> Result { - match DataFrame::new(vec![series]) { - Ok(dataframe) => Ok(NuDataFrame::dataframe_into_value(dataframe, span)), - Err(e) => Err(ShellError::GenericError { - error: "Error creating dataframe".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }), - } - } - - pub fn try_from_iter(iter: T, maybe_schema: Option) -> Result - where - T: Iterator, - { - // Dictionary to store the columnar data extracted from - // the input. During the iteration we check if the values - // have different type - let mut column_values: ColumnMap = IndexMap::new(); - - for value in iter { - match value { - Value::Custom { .. } => return Self::try_from_value(value), - Value::List { vals, .. } => { - let record = vals - .into_iter() - .enumerate() - .map(|(i, val)| (format!("{i}"), val)) - .collect(); - - conversion::insert_record(&mut column_values, record, &maybe_schema)? - } - Value::Record { val: record, .. } => conversion::insert_record( - &mut column_values, - record.into_owned(), - &maybe_schema, - )?, - _ => { - let key = "0".to_string(); - conversion::insert_value(value, key, &mut column_values, &maybe_schema)? - } - } - } - - let df = conversion::from_parsed_columns(column_values)?; - add_missing_columns(df, &maybe_schema, Span::unknown()) - } - - pub fn try_from_series(columns: Vec, span: Span) -> Result { - let dataframe = DataFrame::new(columns).map_err(|e| ShellError::GenericError { - error: "Error creating dataframe".into(), - msg: format!("Unable to create DataFrame: {e}"), - span: Some(span), - help: None, - inner: vec![], - })?; - - Ok(Self::new(false, dataframe)) - } - - pub fn try_from_columns( - columns: Vec, - maybe_schema: Option, - ) -> Result { - let mut column_values: ColumnMap = IndexMap::new(); - - for column in columns { - let name = column.name().to_string(); - for value in column { - conversion::insert_value(value, name.clone(), &mut column_values, &maybe_schema)?; - } - } - - let df = conversion::from_parsed_columns(column_values)?; - add_missing_columns(df, &maybe_schema, Span::unknown()) - } - - pub fn fill_list_nan(list: Vec, list_span: Span, fill: Value) -> Value { - let newlist = list - .into_iter() - .map(|value| { - let span = value.span(); - match value { - Value::Float { val, .. } => { - if val.is_nan() { - fill.clone() - } else { - value - } - } - Value::List { vals, .. } => Self::fill_list_nan(vals, span, fill.clone()), - _ => value, - } - }) - .collect::>(); - Value::list(newlist, list_span) - } - - pub fn columns(&self, span: Span) -> Result, ShellError> { - let height = self.df.height(); - self.df - .get_columns() - .iter() - .map(|col| conversion::create_column(col, 0, height, span)) - .collect::, ShellError>>() - } - - pub fn try_from_value(value: Value) -> Result { - if Self::can_downcast(&value) { - Ok(Self::get_df(value)?) - } else if NuLazyFrame::can_downcast(&value) { - let span = value.span(); - let lazy = NuLazyFrame::try_from_value(value)?; - let df = lazy.collect(span)?; - Ok(df) - } else { - Err(ShellError::CantConvert { - to_type: "lazy or eager dataframe".into(), - from_type: value.get_type().to_string(), - span: value.span(), - help: None, - }) - } - } - - pub fn get_df(value: Value) -> Result { - let span = value.span(); - match value { - Value::Custom { val, .. } => match val.as_any().downcast_ref::() { - Some(df) => Ok(NuDataFrame { - df: df.df.clone(), - from_lazy: false, - }), - None => Err(ShellError::CantConvert { - to_type: "dataframe".into(), - from_type: "non-dataframe".into(), - span, - help: None, - }), - }, - x => Err(ShellError::CantConvert { - to_type: "dataframe".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }), - } - } - - pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span)?; - Self::try_from_value(value) - } - - pub fn can_downcast(value: &Value) -> bool { - if let Value::Custom { val, .. } = value { - val.as_any().downcast_ref::().is_some() - } else { - false - } - } - - pub fn column(&self, column: &str, span: Span) -> Result { - let s = self.df.column(column).map_err(|_| { - let possibilities = self - .df - .get_column_names() - .iter() - .map(|name| name.to_string()) - .collect::>(); - - let option = did_you_mean(&possibilities, column).unwrap_or_else(|| column.to_string()); - ShellError::DidYouMean { - suggestion: option, - span, - } - })?; - - let df = DataFrame::new(vec![s.clone()]).map_err(|e| ShellError::GenericError { - error: "Error creating dataframe".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - Ok(Self { - df, - from_lazy: false, - }) - } - - pub fn is_series(&self) -> bool { - self.df.width() == 1 - } - - pub fn as_series(&self, span: Span) -> Result { - if !self.is_series() { - return Err(ShellError::GenericError { - error: "Error using as series".into(), - msg: "dataframe has more than one column".into(), - span: Some(span), - help: None, - inner: vec![], - }); - } - - let series = self - .df - .get_columns() - .first() - .expect("We have already checked that the width is 1"); - - Ok(series.clone()) - } - - pub fn get_value(&self, row: usize, span: Span) -> Result { - let series = self.as_series(span)?; - let column = conversion::create_column(&series, row, row + 1, span)?; - - if column.len() == 0 { - Err(ShellError::AccessEmptyContent { span }) - } else { - let value = column - .into_iter() - .next() - .expect("already checked there is a value"); - Ok(value) - } - } - - // Print is made out a head and if the dataframe is too large, then a tail - pub fn print(&self, span: Span) -> Result, ShellError> { - let df = &self.df; - let size: usize = 20; - - if df.height() > size { - let sample_size = size / 2; - let mut values = self.head(Some(sample_size), span)?; - conversion::add_separator(&mut values, df, span); - let remaining = df.height() - sample_size; - let tail_size = remaining.min(sample_size); - let mut tail_values = self.tail(Some(tail_size), span)?; - values.append(&mut tail_values); - - Ok(values) - } else { - Ok(self.head(Some(size), span)?) - } - } - - pub fn height(&self) -> usize { - self.df.height() - } - - pub fn head(&self, rows: Option, span: Span) -> Result, ShellError> { - let to_row = rows.unwrap_or(5); - let values = self.to_rows(0, to_row, span)?; - - Ok(values) - } - - pub fn tail(&self, rows: Option, span: Span) -> Result, ShellError> { - let df = &self.df; - let to_row = df.height(); - let size = rows.unwrap_or(DEFAULT_ROWS); - let from_row = to_row.saturating_sub(size); - - let values = self.to_rows(from_row, to_row, span)?; - - Ok(values) - } - - pub fn to_rows( - &self, - from_row: usize, - to_row: usize, - span: Span, - ) -> Result, ShellError> { - let df = &self.df; - let upper_row = to_row.min(df.height()); - - let mut size: usize = 0; - let columns = self - .df - .get_columns() - .iter() - .map( - |col| match conversion::create_column(col, from_row, upper_row, span) { - Ok(col) => { - size = col.len(); - Ok(col) - } - Err(e) => Err(e), - }, - ) - .collect::, ShellError>>()?; - - let mut iterators = columns - .into_iter() - .map(|col| (col.name().to_string(), col.into_iter())) - .collect::)>>(); - - let values = (0..size) - .map(|i| { - let mut record = Record::new(); - - record.push("index", Value::int((i + from_row) as i64, span)); - - for (name, col) in &mut iterators { - record.push(name.clone(), col.next().unwrap_or(Value::nothing(span))); - } - - Value::record(record, span) - }) - .collect::>(); - - Ok(values) - } - - // Dataframes are considered equal if they have the same shape, column name and values - pub fn is_equal(&self, other: &Self) -> Option { - if self.as_ref().width() == 0 { - // checking for empty dataframe - return None; - } - - if self.as_ref().get_column_names() != other.as_ref().get_column_names() { - // checking both dataframes share the same names - return None; - } - - if self.as_ref().height() != other.as_ref().height() { - // checking both dataframes have the same row size - return None; - } - - // sorting dataframe by the first column - let column_names = self.as_ref().get_column_names(); - let first_col = column_names - .first() - .expect("already checked that dataframe is different than 0"); - - // if unable to sort, then unable to compare - let lhs = match self - .as_ref() - .sort(vec![*first_col], SortMultipleOptions::default()) - { - Ok(df) => df, - Err(_) => return None, - }; - - let rhs = match other - .as_ref() - .sort(vec![*first_col], SortMultipleOptions::default()) - { - Ok(df) => df, - Err(_) => return None, - }; - - for name in self.as_ref().get_column_names() { - let self_series = lhs.column(name).expect("name from dataframe names"); - - let other_series = rhs - .column(name) - .expect("already checked that name in other"); - - let self_series = match self_series.dtype() { - // Casting needed to compare other numeric types with nushell numeric type. - // In nushell we only have i64 integer numeric types and any array created - // with nushell untagged primitives will be of type i64 - DataType::UInt32 | DataType::Int32 => match self_series.cast(&DataType::Int64) { - Ok(series) => series, - Err(_) => return None, - }, - _ => self_series.clone(), - }; - - if !self_series.equals(other_series) { - return None; - } - } - - Some(Ordering::Equal) - } - - pub fn schema(&self) -> NuSchema { - NuSchema::new(self.df.schema()) - } -} - -fn add_missing_columns( - df: NuDataFrame, - maybe_schema: &Option, - span: Span, -) -> Result { - // If there are fields that are in the schema, but not in the dataframe - // add them to the dataframe. - if let Some(schema) = maybe_schema { - let fields = df.df.fields(); - let df_field_names: HashSet<&str> = fields.iter().map(|f| f.name().as_str()).collect(); - - let missing: Vec<(&str, &DataType)> = schema - .schema - .iter() - .filter_map(|(name, dtype)| { - let name = name.as_str(); - if !df_field_names.contains(name) { - Some((name, dtype)) - } else { - None - } - }) - .collect(); - - let missing_exprs: Vec = missing - .iter() - .map(|(name, dtype)| lit(Null {}).cast((*dtype).to_owned()).alias(name)) - .collect(); - - let df = if !missing.is_empty() { - let with_columns = df.lazy().with_columns(missing_exprs); - NuLazyFrame::new(true, with_columns).collect(span)? - } else { - df - }; - Ok(df) - } else { - Ok(df) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/operations.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/operations.rs deleted file mode 100644 index ff2f7b7604..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_dataframe/operations.rs +++ /dev/null @@ -1,206 +0,0 @@ -use super::{ - between_values::{between_dataframes, compute_between_series, compute_series_single_value}, - NuDataFrame, -}; -use nu_protocol::{ast::Operator, ShellError, Span, Spanned, Value}; -use polars::prelude::{DataFrame, Series}; - -pub enum Axis { - Row, - Column, -} - -impl NuDataFrame { - pub fn compute_with_value( - &self, - lhs_span: Span, - operator: Operator, - op_span: Span, - right: &Value, - ) -> Result { - let rhs_span = right.span(); - match right { - Value::Custom { val: rhs, .. } => { - let rhs = rhs.as_any().downcast_ref::().ok_or_else(|| { - ShellError::DowncastNotPossible { - msg: "Unable to create dataframe".to_string(), - span: rhs_span, - } - })?; - - match (self.is_series(), rhs.is_series()) { - (true, true) => { - let lhs = &self - .as_series(lhs_span) - .expect("Already checked that is a series"); - let rhs = &rhs - .as_series(rhs_span) - .expect("Already checked that is a series"); - - if lhs.dtype() != rhs.dtype() { - return Err(ShellError::IncompatibleParameters { - left_message: format!("datatype {}", lhs.dtype()), - left_span: lhs_span, - right_message: format!("datatype {}", lhs.dtype()), - right_span: rhs_span, - }); - } - - if lhs.len() != rhs.len() { - return Err(ShellError::IncompatibleParameters { - left_message: format!("len {}", lhs.len()), - left_span: lhs_span, - right_message: format!("len {}", rhs.len()), - right_span: rhs_span, - }); - } - - let op = Spanned { - item: operator, - span: op_span, - }; - - compute_between_series( - op, - &NuDataFrame::default_value(lhs_span), - lhs, - right, - rhs, - ) - } - _ => { - if self.df.height() != rhs.df.height() { - return Err(ShellError::IncompatibleParameters { - left_message: format!("rows {}", self.df.height()), - left_span: lhs_span, - right_message: format!("rows {}", rhs.df.height()), - right_span: rhs_span, - }); - } - - let op = Spanned { - item: operator, - span: op_span, - }; - - between_dataframes( - op, - &NuDataFrame::default_value(lhs_span), - self, - right, - rhs, - ) - } - } - } - _ => { - let op = Spanned { - item: operator, - span: op_span, - }; - - compute_series_single_value(op, &NuDataFrame::default_value(lhs_span), self, right) - } - } - } - - pub fn append_df( - &self, - other: &NuDataFrame, - axis: Axis, - span: Span, - ) -> Result { - match axis { - Axis::Row => { - let mut columns: Vec<&str> = Vec::new(); - - let new_cols = self - .df - .get_columns() - .iter() - .chain(other.df.get_columns()) - .map(|s| { - let name = if columns.contains(&s.name()) { - format!("{}_{}", s.name(), "x") - } else { - columns.push(s.name()); - s.name().to_string() - }; - - let mut series = s.clone(); - series.rename(&name); - series - }) - .collect::>(); - - let df_new = DataFrame::new(new_cols).map_err(|e| ShellError::GenericError { - error: "Error creating dataframe".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - })?; - - Ok(NuDataFrame::new(false, df_new)) - } - Axis::Column => { - if self.df.width() != other.df.width() { - return Err(ShellError::IncompatibleParametersSingle { - msg: "Dataframes with different number of columns".into(), - span, - }); - } - - if !self - .df - .get_column_names() - .iter() - .all(|col| other.df.get_column_names().contains(col)) - { - return Err(ShellError::IncompatibleParametersSingle { - msg: "Dataframes with different columns names".into(), - span, - }); - } - - let new_cols = self - .df - .get_columns() - .iter() - .map(|s| { - let other_col = other - .df - .column(s.name()) - .expect("Already checked that dataframes have same columns"); - - let mut tmp = s.clone(); - let res = tmp.append(other_col); - - match res { - Ok(s) => Ok(s.clone()), - Err(e) => Err({ - ShellError::GenericError { - error: "Error appending dataframe".into(), - msg: format!("Unable to append: {e}"), - span: Some(span), - help: None, - inner: vec![], - } - }), - } - }) - .collect::, ShellError>>()?; - - let df_new = DataFrame::new(new_cols).map_err(|e| ShellError::GenericError { - error: "Error appending dataframe".into(), - msg: format!("Unable to append dataframes: {e}"), - span: Some(span), - help: None, - inner: vec![], - })?; - - Ok(NuDataFrame::new(false, df_new)) - } - } - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/custom_value.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/custom_value.rs deleted file mode 100644 index 7a7f59e648..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/custom_value.rs +++ /dev/null @@ -1,147 +0,0 @@ -use super::NuExpression; -use nu_protocol::{ - ast::{Comparison, Math, Operator}, - CustomValue, ShellError, Span, Type, Value, -}; -use polars::prelude::Expr; -use std::ops::{Add, Div, Mul, Rem, Sub}; - -// CustomValue implementation for NuDataFrame -impl CustomValue for NuExpression { - fn typetag_name(&self) -> &'static str { - "expression" - } - - fn typetag_deserialize(&self) { - unimplemented!("typetag_deserialize") - } - - fn clone_value(&self, span: nu_protocol::Span) -> Value { - let cloned = NuExpression(self.0.clone()); - - Value::custom(Box::new(cloned), span) - } - - fn type_name(&self) -> String { - self.typetag_name().to_string() - } - - fn to_base_value(&self, span: Span) -> Result { - self.to_value(span) - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } - - fn operation( - &self, - lhs_span: Span, - operator: Operator, - op: Span, - right: &Value, - ) -> Result { - compute_with_value(self, lhs_span, operator, op, right) - } -} - -fn compute_with_value( - left: &NuExpression, - lhs_span: Span, - operator: Operator, - op: Span, - right: &Value, -) -> Result { - let rhs_span = right.span(); - match right { - Value::Custom { val: rhs, .. } => { - let rhs = rhs.as_any().downcast_ref::().ok_or_else(|| { - ShellError::DowncastNotPossible { - msg: "Unable to create expression".into(), - span: rhs_span, - } - })?; - - match rhs.as_ref() { - polars::prelude::Expr::Literal(..) => { - with_operator(operator, left, rhs, lhs_span, right.span(), op) - } - _ => Err(ShellError::TypeMismatch { - err_message: "Only literal expressions or number".into(), - span: right.span(), - }), - } - } - _ => { - let rhs = NuExpression::try_from_value(right.clone())?; - with_operator(operator, left, &rhs, lhs_span, right.span(), op) - } - } -} - -fn with_operator( - operator: Operator, - left: &NuExpression, - right: &NuExpression, - lhs_span: Span, - rhs_span: Span, - op_span: Span, -) -> Result { - match operator { - Operator::Math(Math::Plus) => apply_arithmetic(left, right, lhs_span, Add::add), - Operator::Math(Math::Minus) => apply_arithmetic(left, right, lhs_span, Sub::sub), - Operator::Math(Math::Multiply) => apply_arithmetic(left, right, lhs_span, Mul::mul), - Operator::Math(Math::Divide) => apply_arithmetic(left, right, lhs_span, Div::div), - Operator::Math(Math::Modulo) => apply_arithmetic(left, right, lhs_span, Rem::rem), - Operator::Math(Math::FloorDivision) => apply_arithmetic(left, right, lhs_span, Div::div), - Operator::Comparison(Comparison::Equal) => Ok(left - .clone() - .apply_with_expr(right.clone(), Expr::eq) - .into_value(lhs_span)), - Operator::Comparison(Comparison::NotEqual) => Ok(left - .clone() - .apply_with_expr(right.clone(), Expr::neq) - .into_value(lhs_span)), - Operator::Comparison(Comparison::GreaterThan) => Ok(left - .clone() - .apply_with_expr(right.clone(), Expr::gt) - .into_value(lhs_span)), - Operator::Comparison(Comparison::GreaterThanOrEqual) => Ok(left - .clone() - .apply_with_expr(right.clone(), Expr::gt_eq) - .into_value(lhs_span)), - Operator::Comparison(Comparison::LessThan) => Ok(left - .clone() - .apply_with_expr(right.clone(), Expr::lt) - .into_value(lhs_span)), - Operator::Comparison(Comparison::LessThanOrEqual) => Ok(left - .clone() - .apply_with_expr(right.clone(), Expr::lt_eq) - .into_value(lhs_span)), - _ => Err(ShellError::OperatorMismatch { - op_span, - lhs_ty: Type::Custom(left.typetag_name().into()).to_string(), - lhs_span, - rhs_ty: Type::Custom(right.typetag_name().into()).to_string(), - rhs_span, - }), - } -} - -fn apply_arithmetic( - left: &NuExpression, - right: &NuExpression, - span: Span, - f: F, -) -> Result -where - F: Fn(Expr, Expr) -> Expr, -{ - let expr: NuExpression = f(left.as_ref().clone(), right.as_ref().clone()).into(); - - Ok(expr.into_value(span)) -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs deleted file mode 100644 index cee31d7b53..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_expression/mod.rs +++ /dev/null @@ -1,443 +0,0 @@ -mod custom_value; - -use nu_protocol::{record, PipelineData, ShellError, Span, Value}; -use polars::prelude::{col, AggExpr, Expr, Literal}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -// Polars Expression wrapper for Nushell operations -// Object is behind and Option to allow easy implementation of -// the Deserialize trait -#[derive(Default, Clone, Debug)] -pub struct NuExpression(Option); - -// Mocked serialization of the LazyFrame object -impl Serialize for NuExpression { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_none() - } -} - -// Mocked deserialization of the LazyFrame object -impl<'de> Deserialize<'de> for NuExpression { - fn deserialize(_deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Ok(NuExpression::default()) - } -} - -// Referenced access to the real LazyFrame -impl AsRef for NuExpression { - fn as_ref(&self) -> &polars::prelude::Expr { - // The only case when there cannot be an expr is if it is created - // using the default function or if created by deserializing something - self.0.as_ref().expect("there should always be a frame") - } -} - -impl AsMut for NuExpression { - fn as_mut(&mut self) -> &mut polars::prelude::Expr { - // The only case when there cannot be an expr is if it is created - // using the default function or if created by deserializing something - self.0.as_mut().expect("there should always be a frame") - } -} - -impl From for NuExpression { - fn from(expr: Expr) -> Self { - Self(Some(expr)) - } -} - -impl NuExpression { - pub fn into_value(self, span: Span) -> Value { - Value::custom(Box::new(self), span) - } - - pub fn try_from_value(value: Value) -> Result { - let span = value.span(); - match value { - Value::Custom { val, .. } => match val.as_any().downcast_ref::() { - Some(expr) => Ok(NuExpression(expr.0.clone())), - None => Err(ShellError::CantConvert { - to_type: "lazy expression".into(), - from_type: "non-dataframe".into(), - span, - help: None, - }), - }, - Value::String { val, .. } => Ok(val.lit().into()), - Value::Int { val, .. } => Ok(val.lit().into()), - Value::Bool { val, .. } => Ok(val.lit().into()), - Value::Float { val, .. } => Ok(val.lit().into()), - x => Err(ShellError::CantConvert { - to_type: "lazy expression".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }), - } - } - - pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span)?; - Self::try_from_value(value) - } - - pub fn can_downcast(value: &Value) -> bool { - match value { - Value::Custom { val, .. } => val.as_any().downcast_ref::().is_some(), - Value::List { vals, .. } => vals.iter().all(Self::can_downcast), - Value::String { .. } | Value::Int { .. } | Value::Bool { .. } | Value::Float { .. } => { - true - } - _ => false, - } - } - - pub fn into_polars(self) -> Expr { - self.0.expect("Expression cannot be none to convert") - } - - pub fn apply_with_expr(self, other: NuExpression, f: F) -> Self - where - F: Fn(Expr, Expr) -> Expr, - { - let expr = self.0.expect("Lazy expression must not be empty to apply"); - let other = other.0.expect("Lazy expression must not be empty to apply"); - - f(expr, other).into() - } - - pub fn to_value(&self, span: Span) -> Result { - expr_to_value(self.as_ref(), span) - } - - // Convenient function to extract multiple Expr that could be inside a nushell Value - pub fn extract_exprs(value: Value) -> Result, ShellError> { - ExtractedExpr::extract_exprs(value).map(ExtractedExpr::into_exprs) - } -} - -#[derive(Debug)] -// Enum to represent the parsing of the expressions from Value -enum ExtractedExpr { - Single(Expr), - List(Vec), -} - -impl ExtractedExpr { - fn into_exprs(self) -> Vec { - match self { - Self::Single(expr) => vec![expr], - Self::List(expressions) => expressions - .into_iter() - .flat_map(ExtractedExpr::into_exprs) - .collect(), - } - } - - fn extract_exprs(value: Value) -> Result { - match value { - Value::String { val, .. } => Ok(ExtractedExpr::Single(col(val.as_str()))), - Value::Custom { .. } => NuExpression::try_from_value(value) - .map(NuExpression::into_polars) - .map(ExtractedExpr::Single), - Value::List { vals, .. } => vals - .into_iter() - .map(Self::extract_exprs) - .collect::, ShellError>>() - .map(ExtractedExpr::List), - x => Err(ShellError::CantConvert { - to_type: "expression".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }), - } - } -} - -pub fn expr_to_value(expr: &Expr, span: Span) -> Result { - match expr { - Expr::Alias(expr, alias) => Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "alias" => Value::string(alias.as_ref(), span), - }, - span, - )), - Expr::Column(name) => Ok(Value::record( - record! { - "expr" => Value::string("column", span), - "value" => Value::string(name.to_string(), span), - }, - span, - )), - Expr::Columns(columns) => { - let value = columns.iter().map(|col| Value::string(col, span)).collect(); - Ok(Value::record( - record! { - "expr" => Value::string("columns", span), - "value" => Value::list(value, span), - }, - span, - )) - } - Expr::Literal(literal) => Ok(Value::record( - record! { - "expr" => Value::string("literal", span), - "value" => Value::string(format!("{literal:?}"), span), - }, - span, - )), - Expr::BinaryExpr { left, op, right } => Ok(Value::record( - record! { - "left" => expr_to_value(left, span)?, - "op" => Value::string(format!("{op:?}"), span), - "right" => expr_to_value(right, span)?, - }, - span, - )), - Expr::Ternary { - predicate, - truthy, - falsy, - } => Ok(Value::record( - record! { - "predicate" => expr_to_value(predicate.as_ref(), span)?, - "truthy" => expr_to_value(truthy.as_ref(), span)?, - "falsy" => expr_to_value(falsy.as_ref(), span)?, - }, - span, - )), - Expr::Agg(agg_expr) => { - let value = match agg_expr { - AggExpr::Min { input: expr, .. } - | AggExpr::Max { input: expr, .. } - | AggExpr::Median(expr) - | AggExpr::NUnique(expr) - | AggExpr::First(expr) - | AggExpr::Last(expr) - | AggExpr::Mean(expr) - | AggExpr::Implode(expr) - | AggExpr::Count(expr, _) - | AggExpr::Sum(expr) - | AggExpr::AggGroups(expr) - | AggExpr::Std(expr, _) - | AggExpr::Var(expr, _) => expr_to_value(expr.as_ref(), span), - AggExpr::Quantile { - expr, - quantile, - interpol, - } => Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "quantile" => expr_to_value(quantile.as_ref(), span)?, - "interpol" => Value::string(format!("{interpol:?}"), span), - }, - span, - )), - }; - - Ok(Value::record( - record! { - "expr" => Value::string("agg", span), - "value" => value?, - }, - span, - )) - } - Expr::Len => Ok(Value::record( - record! { "expr" => Value::string("count", span) }, - span, - )), - Expr::Wildcard => Ok(Value::record( - record! { "expr" => Value::string("wildcard", span) }, - span, - )), - Expr::Explode(expr) => Ok(Value::record( - record! { "expr" => expr_to_value(expr.as_ref(), span)? }, - span, - )), - Expr::KeepName(expr) => Ok(Value::record( - record! { "expr" => expr_to_value(expr.as_ref(), span)? }, - span, - )), - Expr::Nth(i) => Ok(Value::record( - record! { "expr" => Value::int(*i, span) }, - span, - )), - Expr::DtypeColumn(dtypes) => { - let vals = dtypes - .iter() - .map(|d| Value::string(format!("{d}"), span)) - .collect(); - - Ok(Value::list(vals, span)) - } - Expr::Sort { expr, options } => Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "options" => Value::string(format!("{options:?}"), span), - }, - span, - )), - Expr::Cast { - expr, - data_type, - strict, - } => Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "dtype" => Value::string(format!("{data_type:?}"), span), - "strict" => Value::bool(*strict, span), - }, - span, - )), - Expr::Gather { - expr, - idx, - returns_scalar: _, - } => Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "idx" => expr_to_value(idx.as_ref(), span)?, - }, - span, - )), - Expr::SortBy { - expr, - by, - sort_options, - } => { - let by: Result, ShellError> = - by.iter().map(|b| expr_to_value(b, span)).collect(); - let descending: Vec = sort_options - .descending - .iter() - .map(|r| Value::bool(*r, span)) - .collect(); - - Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "by" => Value::list(by?, span), - "descending" => Value::list(descending, span), - }, - span, - )) - } - Expr::Filter { input, by } => Ok(Value::record( - record! { - "input" => expr_to_value(input.as_ref(), span)?, - "by" => expr_to_value(by.as_ref(), span)?, - }, - span, - )), - Expr::Slice { - input, - offset, - length, - } => Ok(Value::record( - record! { - "input" => expr_to_value(input.as_ref(), span)?, - "offset" => expr_to_value(offset.as_ref(), span)?, - "length" => expr_to_value(length.as_ref(), span)?, - }, - span, - )), - Expr::Exclude(expr, excluded) => { - let excluded = excluded - .iter() - .map(|e| Value::string(format!("{e:?}"), span)) - .collect(); - - Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "excluded" => Value::list(excluded, span), - }, - span, - )) - } - Expr::RenameAlias { expr, function } => Ok(Value::record( - record! { - "expr" => expr_to_value(expr.as_ref(), span)?, - "function" => Value::string(format!("{function:?}"), span), - }, - span, - )), - Expr::AnonymousFunction { - input, - function, - output_type, - options, - } => { - let input: Result, ShellError> = - input.iter().map(|e| expr_to_value(e, span)).collect(); - Ok(Value::record( - record! { - "input" => Value::list(input?, span), - "function" => Value::string(format!("{function:?}"), span), - "output_type" => Value::string(format!("{output_type:?}"), span), - "options" => Value::string(format!("{options:?}"), span), - }, - span, - )) - } - Expr::Function { - input, - function, - options, - } => { - let input: Result, ShellError> = - input.iter().map(|e| expr_to_value(e, span)).collect(); - Ok(Value::record( - record! { - "input" => Value::list(input?, span), - "function" => Value::string(format!("{function:?}"), span), - "options" => Value::string(format!("{options:?}"), span), - }, - span, - )) - } - Expr::Window { - function, - partition_by, - options, - } => { - let partition_by: Result, ShellError> = partition_by - .iter() - .map(|e| expr_to_value(e, span)) - .collect(); - - Ok(Value::record( - record! { - "function" => expr_to_value(function, span)?, - "partition_by" => Value::list(partition_by?, span), - "options" => Value::string(format!("{options:?}"), span), - }, - span, - )) - } - Expr::SubPlan(_, _) => Err(ShellError::UnsupportedInput { - msg: "Expressions of type SubPlan are not yet supported".to_string(), - input: format!("Expression is {expr:?}"), - msg_span: span, - input_span: Span::unknown(), - }), - // the parameter polars_plan::dsl::selector::Selector is not publicly exposed. - // I am not sure what we can meaningfully do with this at this time. - Expr::Selector(_) => Err(ShellError::UnsupportedInput { - msg: "Expressions of type Selector to Nu Values is not yet supported".to_string(), - input: format!("Expression is {expr:?}"), - msg_span: span, - input_span: Span::unknown(), - }), - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/custom_value.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/custom_value.rs deleted file mode 100644 index f747ae4d18..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/custom_value.rs +++ /dev/null @@ -1,50 +0,0 @@ -use super::NuLazyFrame; -use nu_protocol::{record, CustomValue, ShellError, Span, Value}; - -// CustomValue implementation for NuDataFrame -impl CustomValue for NuLazyFrame { - fn typetag_name(&self) -> &'static str { - "lazyframe" - } - - fn typetag_deserialize(&self) { - unimplemented!("typetag_deserialize") - } - - fn clone_value(&self, span: nu_protocol::Span) -> Value { - let cloned = NuLazyFrame { - lazy: self.lazy.clone(), - from_eager: self.from_eager, - schema: self.schema.clone(), - }; - - Value::custom(Box::new(cloned), span) - } - - fn type_name(&self) -> String { - self.typetag_name().to_string() - } - - fn to_base_value(&self, span: Span) -> Result { - let optimized_plan = self - .as_ref() - .describe_optimized_plan() - .unwrap_or_else(|_| "".to_string()); - - Ok(Value::record( - record! { - "plan" => Value::string(self.as_ref().describe_plan(), span), - "optimized_plan" => Value::string(optimized_plan, span), - }, - span, - )) - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs deleted file mode 100644 index 355516d340..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazyframe/mod.rs +++ /dev/null @@ -1,188 +0,0 @@ -mod custom_value; - -use super::{NuDataFrame, NuExpression}; -use core::fmt; -use nu_protocol::{PipelineData, ShellError, Span, Value}; -use polars::prelude::{Expr, IntoLazy, LazyFrame, Schema}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -// Lazyframe wrapper for Nushell operations -// Polars LazyFrame is behind and Option to allow easy implementation of -// the Deserialize trait -#[derive(Default)] -pub struct NuLazyFrame { - pub lazy: Option, - pub schema: Option, - pub from_eager: bool, -} - -// Mocked serialization of the LazyFrame object -impl Serialize for NuLazyFrame { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_none() - } -} - -// Mocked deserialization of the LazyFrame object -impl<'de> Deserialize<'de> for NuLazyFrame { - fn deserialize(_deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Ok(NuLazyFrame::default()) - } -} - -impl fmt::Debug for NuLazyFrame { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "NuLazyframe") - } -} - -// Referenced access to the real LazyFrame -impl AsRef for NuLazyFrame { - fn as_ref(&self) -> &polars::prelude::LazyFrame { - // The only case when there cannot be a lazy frame is if it is created - // using the default function or if created by deserializing something - self.lazy.as_ref().expect("there should always be a frame") - } -} - -impl AsMut for NuLazyFrame { - fn as_mut(&mut self) -> &mut polars::prelude::LazyFrame { - // The only case when there cannot be a lazy frame is if it is created - // using the default function or if created by deserializing something - self.lazy.as_mut().expect("there should always be a frame") - } -} - -impl From for NuLazyFrame { - fn from(lazy_frame: LazyFrame) -> Self { - Self { - lazy: Some(lazy_frame), - from_eager: false, - schema: None, - } - } -} - -impl NuLazyFrame { - pub fn new(from_eager: bool, lazy: LazyFrame) -> Self { - Self { - lazy: Some(lazy), - from_eager, - schema: None, - } - } - - pub fn from_dataframe(df: NuDataFrame) -> Self { - let lazy = df.as_ref().clone().lazy(); - Self { - lazy: Some(lazy), - from_eager: true, - schema: Some(df.as_ref().schema()), - } - } - - pub fn into_value(self, span: Span) -> Result { - if self.from_eager { - let df = self.collect(span)?; - Ok(Value::custom(Box::new(df), span)) - } else { - Ok(Value::custom(Box::new(self), span)) - } - } - - pub fn into_polars(self) -> LazyFrame { - self.lazy.expect("lazyframe cannot be none to convert") - } - - pub fn collect(self, span: Span) -> Result { - self.lazy - .expect("No empty lazy for collect") - .collect() - .map_err(|e| ShellError::GenericError { - error: "Error collecting lazy frame".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }) - .map(|df| NuDataFrame { - df, - from_lazy: !self.from_eager, - }) - } - - pub fn try_from_value(value: Value) -> Result { - if Self::can_downcast(&value) { - Ok(Self::get_lazy_df(value)?) - } else if NuDataFrame::can_downcast(&value) { - let df = NuDataFrame::try_from_value(value)?; - Ok(NuLazyFrame::from_dataframe(df)) - } else { - Err(ShellError::CantConvert { - to_type: "lazy or eager dataframe".into(), - from_type: value.get_type().to_string(), - span: value.span(), - help: None, - }) - } - } - - pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span)?; - Self::try_from_value(value) - } - - pub fn get_lazy_df(value: Value) -> Result { - let span = value.span(); - match value { - Value::Custom { val, .. } => match val.as_any().downcast_ref::() { - Some(expr) => Ok(Self { - lazy: expr.lazy.clone(), - from_eager: false, - schema: None, - }), - None => Err(ShellError::CantConvert { - to_type: "lazy frame".into(), - from_type: "non-dataframe".into(), - span, - help: None, - }), - }, - x => Err(ShellError::CantConvert { - to_type: "lazy frame".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }), - } - } - - pub fn can_downcast(value: &Value) -> bool { - if let Value::Custom { val, .. } = value { - val.as_any().downcast_ref::().is_some() - } else { - false - } - } - - pub fn apply_with_expr(self, expr: NuExpression, f: F) -> Self - where - F: Fn(LazyFrame, Expr) -> LazyFrame, - { - let df = self.lazy.expect("Lazy frame must not be empty to apply"); - let expr = expr.into_polars(); - let new_frame = f(df, expr); - - Self { - from_eager: self.from_eager, - lazy: Some(new_frame), - schema: None, - } - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/custom_value.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/custom_value.rs deleted file mode 100644 index 6ac6cc6046..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/custom_value.rs +++ /dev/null @@ -1,44 +0,0 @@ -use super::NuLazyGroupBy; -use nu_protocol::{record, CustomValue, ShellError, Span, Value}; - -// CustomValue implementation for NuDataFrame -impl CustomValue for NuLazyGroupBy { - fn typetag_name(&self) -> &'static str { - "lazygroupby" - } - - fn typetag_deserialize(&self) { - unimplemented!("typetag_deserialize") - } - - fn clone_value(&self, span: nu_protocol::Span) -> Value { - let cloned = NuLazyGroupBy { - group_by: self.group_by.clone(), - schema: self.schema.clone(), - from_eager: self.from_eager, - }; - - Value::custom(Box::new(cloned), span) - } - - fn type_name(&self) -> String { - self.typetag_name().to_string() - } - - fn to_base_value(&self, span: Span) -> Result { - Ok(Value::record( - record! { - "LazyGroupBy" => Value::string("apply aggregation to complete execution plan", span) - }, - span, - )) - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs deleted file mode 100644 index e1bcb30069..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_lazygroupby/mod.rs +++ /dev/null @@ -1,113 +0,0 @@ -mod custom_value; - -use core::fmt; -use nu_protocol::{PipelineData, ShellError, Span, Value}; -use polars::prelude::{LazyGroupBy, Schema}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -// Lazyframe wrapper for Nushell operations -// Polars LazyFrame is behind and Option to allow easy implementation of -// the Deserialize trait -#[derive(Default)] -pub struct NuLazyGroupBy { - pub group_by: Option, - pub schema: Option, - pub from_eager: bool, -} - -// Mocked serialization of the LazyFrame object -impl Serialize for NuLazyGroupBy { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_none() - } -} - -// Mocked deserialization of the LazyFrame object -impl<'de> Deserialize<'de> for NuLazyGroupBy { - fn deserialize(_deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Ok(NuLazyGroupBy::default()) - } -} - -impl fmt::Debug for NuLazyGroupBy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "NuLazyGroupBy") - } -} - -// Referenced access to the real LazyFrame -impl AsRef for NuLazyGroupBy { - fn as_ref(&self) -> &polars::prelude::LazyGroupBy { - // The only case when there cannot be a lazy frame is if it is created - // using the default function or if created by deserializing something - self.group_by - .as_ref() - .expect("there should always be a frame") - } -} - -impl AsMut for NuLazyGroupBy { - fn as_mut(&mut self) -> &mut polars::prelude::LazyGroupBy { - // The only case when there cannot be a lazy frame is if it is created - // using the default function or if created by deserializing something - self.group_by - .as_mut() - .expect("there should always be a frame") - } -} - -impl From for NuLazyGroupBy { - fn from(group_by: LazyGroupBy) -> Self { - Self { - group_by: Some(group_by), - from_eager: false, - schema: None, - } - } -} - -impl NuLazyGroupBy { - pub fn into_value(self, span: Span) -> Value { - Value::custom(Box::new(self), span) - } - - pub fn into_polars(self) -> LazyGroupBy { - self.group_by.expect("GroupBy cannot be none to convert") - } - - pub fn try_from_value(value: Value) -> Result { - let span = value.span(); - match value { - Value::Custom { val, .. } => match val.as_any().downcast_ref::() { - Some(group) => Ok(Self { - group_by: group.group_by.clone(), - schema: group.schema.clone(), - from_eager: group.from_eager, - }), - None => Err(ShellError::CantConvert { - to_type: "lazy groupby".into(), - from_type: "custom value".into(), - span, - help: None, - }), - }, - x => Err(ShellError::CantConvert { - to_type: "lazy groupby".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }), - } - } - - pub fn try_from_pipeline(input: PipelineData, span: Span) -> Result { - let value = input.into_value(span)?; - Self::try_from_value(value) - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_schema.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_schema.rs deleted file mode 100644 index 3c2f689b85..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_schema.rs +++ /dev/null @@ -1,376 +0,0 @@ -use nu_protocol::{ShellError, Span, Value}; -use polars::prelude::{DataType, Field, Schema, SchemaRef, TimeUnit}; -use std::sync::Arc; - -#[derive(Debug, Clone)] -pub struct NuSchema { - pub schema: SchemaRef, -} - -impl NuSchema { - pub fn new(schema: Schema) -> Self { - Self { - schema: Arc::new(schema), - } - } -} - -impl TryFrom<&Value> for NuSchema { - type Error = ShellError; - fn try_from(value: &Value) -> Result { - let schema = value_to_schema(value, Span::unknown())?; - Ok(Self::new(schema)) - } -} - -impl From for Value { - fn from(schema: NuSchema) -> Self { - fields_to_value(schema.schema.iter_fields(), Span::unknown()) - } -} - -impl From for SchemaRef { - fn from(val: NuSchema) -> Self { - Arc::clone(&val.schema) - } -} - -fn fields_to_value(fields: impl Iterator, span: Span) -> Value { - let record = fields - .map(|field| { - let col = field.name().to_string(); - let val = dtype_to_value(field.data_type(), span); - (col, val) - }) - .collect(); - - Value::record(record, Span::unknown()) -} - -fn dtype_to_value(dtype: &DataType, span: Span) -> Value { - match dtype { - DataType::Struct(fields) => fields_to_value(fields.iter().cloned(), span), - _ => Value::string(dtype.to_string().replace('[', "<").replace(']', ">"), span), - } -} - -fn value_to_schema(value: &Value, span: Span) -> Result { - let fields = value_to_fields(value, span)?; - let schema = Schema::from_iter(fields); - Ok(schema) -} - -fn value_to_fields(value: &Value, span: Span) -> Result, ShellError> { - let fields = value - .as_record()? - .into_iter() - .map(|(col, val)| match val { - Value::Record { .. } => { - let fields = value_to_fields(val, span)?; - let dtype = DataType::Struct(fields); - Ok(Field::new(col, dtype)) - } - _ => { - let dtype = str_to_dtype(&val.coerce_string()?, span)?; - Ok(Field::new(col, dtype)) - } - }) - .collect::, ShellError>>()?; - Ok(fields) -} - -pub fn str_to_dtype(dtype: &str, span: Span) -> Result { - match dtype { - "bool" => Ok(DataType::Boolean), - "u8" => Ok(DataType::UInt8), - "u16" => Ok(DataType::UInt16), - "u32" => Ok(DataType::UInt32), - "u64" => Ok(DataType::UInt64), - "i8" => Ok(DataType::Int8), - "i16" => Ok(DataType::Int16), - "i32" => Ok(DataType::Int32), - "i64" => Ok(DataType::Int64), - "f32" => Ok(DataType::Float32), - "f64" => Ok(DataType::Float64), - "str" => Ok(DataType::String), - "binary" => Ok(DataType::Binary), - "date" => Ok(DataType::Date), - "time" => Ok(DataType::Time), - "null" => Ok(DataType::Null), - "unknown" => Ok(DataType::Unknown), - "object" => Ok(DataType::Object("unknown", None)), - _ if dtype.starts_with("list") => { - let dtype = dtype - .trim_start_matches("list") - .trim_start_matches('<') - .trim_end_matches('>') - .trim(); - let dtype = str_to_dtype(dtype, span)?; - Ok(DataType::List(Box::new(dtype))) - } - _ if dtype.starts_with("datetime") => { - let dtype = dtype - .trim_start_matches("datetime") - .trim_start_matches('<') - .trim_end_matches('>'); - let mut split = dtype.split(','); - let next = split - .next() - .ok_or_else(|| ShellError::GenericError { - error: "Invalid polars data type".into(), - msg: "Missing time unit".into(), - span: Some(span), - help: None, - inner: vec![], - })? - .trim(); - let time_unit = str_to_time_unit(next, span)?; - let next = split - .next() - .ok_or_else(|| ShellError::GenericError { - error: "Invalid polars data type".into(), - msg: "Missing time zone".into(), - span: Some(span), - help: None, - inner: vec![], - })? - .trim(); - let timezone = if "*" == next { - None - } else { - Some(next.to_string()) - }; - Ok(DataType::Datetime(time_unit, timezone)) - } - _ if dtype.starts_with("duration") => { - let inner = dtype.trim_start_matches("duration<").trim_end_matches('>'); - let next = inner - .split(',') - .next() - .ok_or_else(|| ShellError::GenericError { - error: "Invalid polars data type".into(), - msg: "Missing time unit".into(), - span: Some(span), - help: None, - inner: vec![], - })? - .trim(); - let time_unit = str_to_time_unit(next, span)?; - Ok(DataType::Duration(time_unit)) - } - _ => Err(ShellError::GenericError { - error: "Invalid polars data type".into(), - msg: format!("Unknown type: {dtype}"), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -fn str_to_time_unit(ts_string: &str, span: Span) -> Result { - match ts_string { - "ms" => Ok(TimeUnit::Milliseconds), - "us" | "μs" => Ok(TimeUnit::Microseconds), - "ns" => Ok(TimeUnit::Nanoseconds), - _ => Err(ShellError::GenericError { - error: "Invalid polars data type".into(), - msg: "Invalid time unit".into(), - span: Some(span), - help: None, - inner: vec![], - }), - } -} - -#[cfg(test)] -mod test { - - use nu_protocol::record; - - use super::*; - - #[test] - fn test_value_to_schema() { - let address = record! { - "street" => Value::test_string("str"), - "city" => Value::test_string("str"), - }; - - let value = Value::test_record(record! { - "name" => Value::test_string("str"), - "age" => Value::test_string("i32"), - "address" => Value::test_record(address) - }); - - let schema = value_to_schema(&value, Span::unknown()).unwrap(); - let expected = Schema::from_iter(vec![ - Field::new("name", DataType::String), - Field::new("age", DataType::Int32), - Field::new( - "address", - DataType::Struct(vec![ - Field::new("street", DataType::String), - Field::new("city", DataType::String), - ]), - ), - ]); - assert_eq!(schema, expected); - } - - #[test] - fn test_dtype_str_to_schema_simple_types() { - let dtype = "bool"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Boolean; - assert_eq!(schema, expected); - - let dtype = "u8"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::UInt8; - assert_eq!(schema, expected); - - let dtype = "u16"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::UInt16; - assert_eq!(schema, expected); - - let dtype = "u32"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::UInt32; - assert_eq!(schema, expected); - - let dtype = "u64"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::UInt64; - assert_eq!(schema, expected); - - let dtype = "i8"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Int8; - assert_eq!(schema, expected); - - let dtype = "i16"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Int16; - assert_eq!(schema, expected); - - let dtype = "i32"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Int32; - assert_eq!(schema, expected); - - let dtype = "i64"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Int64; - assert_eq!(schema, expected); - - let dtype = "str"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::String; - assert_eq!(schema, expected); - - let dtype = "binary"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Binary; - assert_eq!(schema, expected); - - let dtype = "date"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Date; - assert_eq!(schema, expected); - - let dtype = "time"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Time; - assert_eq!(schema, expected); - - let dtype = "null"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Null; - assert_eq!(schema, expected); - - let dtype = "unknown"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Unknown; - assert_eq!(schema, expected); - - let dtype = "object"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Object("unknown", None); - assert_eq!(schema, expected); - } - - #[test] - fn test_dtype_str_schema_datetime() { - let dtype = "datetime"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Datetime(TimeUnit::Milliseconds, None); - assert_eq!(schema, expected); - - let dtype = "datetime"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Datetime(TimeUnit::Microseconds, None); - assert_eq!(schema, expected); - - let dtype = "datetime<μs, *>"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Datetime(TimeUnit::Microseconds, None); - assert_eq!(schema, expected); - - let dtype = "datetime"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Datetime(TimeUnit::Nanoseconds, None); - assert_eq!(schema, expected); - - let dtype = "datetime"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Datetime(TimeUnit::Milliseconds, Some("UTC".into())); - assert_eq!(schema, expected); - - let dtype = "invalid"; - let schema = str_to_dtype(dtype, Span::unknown()); - assert!(schema.is_err()) - } - - #[test] - fn test_dtype_str_schema_duration() { - let dtype = "duration"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Duration(TimeUnit::Milliseconds); - assert_eq!(schema, expected); - - let dtype = "duration"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Duration(TimeUnit::Microseconds); - assert_eq!(schema, expected); - - let dtype = "duration<μs>"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Duration(TimeUnit::Microseconds); - assert_eq!(schema, expected); - - let dtype = "duration"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::Duration(TimeUnit::Nanoseconds); - assert_eq!(schema, expected); - } - - #[test] - fn test_dtype_str_to_schema_list_types() { - let dtype = "list"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::List(Box::new(DataType::Int32)); - assert_eq!(schema, expected); - - let dtype = "list>"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::List(Box::new(DataType::Duration(TimeUnit::Milliseconds))); - assert_eq!(schema, expected); - - let dtype = "list>"; - let schema = str_to_dtype(dtype, Span::unknown()).unwrap(); - let expected = DataType::List(Box::new(DataType::Datetime(TimeUnit::Milliseconds, None))); - assert_eq!(schema, expected); - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/custom_value.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/custom_value.rs deleted file mode 100644 index e2b73bcef1..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/custom_value.rs +++ /dev/null @@ -1,41 +0,0 @@ -use super::NuWhen; -use nu_protocol::{CustomValue, ShellError, Span, Value}; - -// CustomValue implementation for NuDataFrame -impl CustomValue for NuWhen { - fn typetag_name(&self) -> &'static str { - "when" - } - - fn typetag_deserialize(&self) { - unimplemented!("typetag_deserialize") - } - - fn clone_value(&self, span: nu_protocol::Span) -> Value { - let cloned = self.clone(); - - Value::custom(Box::new(cloned), span) - } - - fn type_name(&self) -> String { - self.typetag_name().to_string() - } - - fn to_base_value(&self, span: Span) -> Result { - let val: String = match self { - NuWhen::Then(_) => "whenthen".into(), - NuWhen::ChainedThen(_) => "whenthenthen".into(), - }; - - let value = Value::string(val, span); - Ok(value) - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn std::any::Any { - self - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/mod.rs b/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/mod.rs deleted file mode 100644 index b33cde7483..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/nu_when/mod.rs +++ /dev/null @@ -1,77 +0,0 @@ -mod custom_value; - -use core::fmt; -use nu_protocol::{ShellError, Span, Value}; -use polars::prelude::{col, when, ChainedThen, Then}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; - -#[derive(Clone)] -pub enum NuWhen { - Then(Box), - ChainedThen(ChainedThen), -} - -// Mocked serialization of the LazyFrame object -impl Serialize for NuWhen { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_none() - } -} - -// Mocked deserialization of the LazyFrame object -impl<'de> Deserialize<'de> for NuWhen { - fn deserialize(_deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Ok(NuWhen::Then(Box::new(when(col("a")).then(col("b"))))) - } -} - -impl fmt::Debug for NuWhen { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "NuWhen") - } -} - -impl From for NuWhen { - fn from(then: Then) -> Self { - NuWhen::Then(Box::new(then)) - } -} - -impl From for NuWhen { - fn from(chained_when: ChainedThen) -> Self { - NuWhen::ChainedThen(chained_when) - } -} - -impl NuWhen { - pub fn into_value(self, span: Span) -> Value { - Value::custom(Box::new(self), span) - } - - pub fn try_from_value(value: Value) -> Result { - let span = value.span(); - match value { - Value::Custom { val, .. } => match val.as_any().downcast_ref::() { - Some(expr) => Ok(expr.clone()), - None => Err(ShellError::CantConvert { - to_type: "when expression".into(), - from_type: "non when expression".into(), - span, - help: None, - }), - }, - x => Err(ShellError::CantConvert { - to_type: "when expression".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }), - } - } -} diff --git a/crates/nu-cmd-dataframe/src/dataframe/values/utils.rs b/crates/nu-cmd-dataframe/src/dataframe/values/utils.rs deleted file mode 100644 index 0dc43399a3..0000000000 --- a/crates/nu-cmd-dataframe/src/dataframe/values/utils.rs +++ /dev/null @@ -1,86 +0,0 @@ -use nu_protocol::{ShellError, Span, Spanned, Value}; - -// Default value used when selecting rows from dataframe -pub const DEFAULT_ROWS: usize = 5; - -// Converts a Vec to a Vec> with a Span marking the whole -// location of the columns for error referencing -pub(crate) fn convert_columns( - columns: Vec, - span: Span, -) -> Result<(Vec>, Span), ShellError> { - // First column span - let mut col_span = columns - .first() - .ok_or_else(|| ShellError::GenericError { - error: "Empty column list".into(), - msg: "Empty list found for command".into(), - span: Some(span), - help: None, - inner: vec![], - }) - .map(|v| v.span())?; - - let res = columns - .into_iter() - .map(|value| { - let span = value.span(); - match value { - Value::String { val, .. } => { - col_span = col_span.merge(span); - Ok(Spanned { item: val, span }) - } - _ => Err(ShellError::GenericError { - error: "Incorrect column format".into(), - msg: "Only string as column name".into(), - span: Some(span), - help: None, - inner: vec![], - }), - } - }) - .collect::>, _>>()?; - - Ok((res, col_span)) -} - -// Converts a Vec to a Vec with a Span marking the whole -// location of the columns for error referencing -pub(crate) fn convert_columns_string( - columns: Vec, - span: Span, -) -> Result<(Vec, Span), ShellError> { - // First column span - let mut col_span = columns - .first() - .ok_or_else(|| ShellError::GenericError { - error: "Empty column list".into(), - msg: "Empty list found for command".into(), - span: Some(span), - help: None, - inner: vec![], - }) - .map(|v| v.span())?; - - let res = columns - .into_iter() - .map(|value| { - let span = value.span(); - match value { - Value::String { val, .. } => { - col_span = col_span.merge(span); - Ok(val) - } - _ => Err(ShellError::GenericError { - error: "Incorrect column format".into(), - msg: "Only string as column name".into(), - span: Some(span), - help: None, - inner: vec![], - }), - } - }) - .collect::, _>>()?; - - Ok((res, col_span)) -} diff --git a/crates/nu-cmd-dataframe/src/lib.rs b/crates/nu-cmd-dataframe/src/lib.rs deleted file mode 100644 index 7e7c8014c6..0000000000 --- a/crates/nu-cmd-dataframe/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[cfg(feature = "dataframe")] -pub mod dataframe; -#[cfg(feature = "dataframe")] -pub use dataframe::*; diff --git a/crates/nu-cmd-extra/Cargo.toml b/crates/nu-cmd-extra/Cargo.toml index 4c3dce09c7..8609adb44c 100644 --- a/crates/nu-cmd-extra/Cargo.toml +++ b/crates/nu-cmd-extra/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" license = "MIT" name = "nu-cmd-extra" repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-extra" -version = "0.93.1" +version = "0.95.1" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -13,13 +13,13 @@ version = "0.93.1" bench = false [dependencies] -nu-cmd-base = { path = "../nu-cmd-base", version = "0.93.1" } -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-json = { version = "0.93.1", path = "../nu-json" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-pretty-hex = { version = "0.93.1", path = "../nu-pretty-hex" } -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } -nu-utils = { path = "../nu-utils", version = "0.93.1" } +nu-cmd-base = { path = "../nu-cmd-base", version = "0.95.1" } +nu-engine = { path = "../nu-engine", version = "0.95.1" } +nu-json = { version = "0.95.1", path = "../nu-json" } +nu-parser = { path = "../nu-parser", version = "0.95.1" } +nu-pretty-hex = { version = "0.95.1", path = "../nu-pretty-hex" } +nu-protocol = { path = "../nu-protocol", version = "0.95.1" } +nu-utils = { path = "../nu-utils", version = "0.95.1" } # Potential dependencies for extras heck = { workspace = true } @@ -32,11 +32,7 @@ serde_urlencoded = { workspace = true } v_htmlescape = { workspace = true } itertools = { workspace = true } -[features] -extra = ["default"] -default = [] - [dev-dependencies] -nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.1" } -nu-command = { path = "../nu-command", version = "0.93.1" } -nu-test-support = { path = "../nu-test-support", version = "0.93.1" } +nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.95.1" } +nu-command = { path = "../nu-command", version = "0.95.1" } +nu-test-support = { path = "../nu-test-support", version = "0.95.1" } \ No newline at end of file diff --git a/crates/nu-cmd-extra/src/extra/bits/and.rs b/crates/nu-cmd-extra/src/extra/bits/and.rs index 538cf6e60f..234b4e5cc1 100644 --- a/crates/nu-cmd-extra/src/extra/bits/and.rs +++ b/crates/nu-cmd-extra/src/extra/bits/and.rs @@ -79,7 +79,7 @@ impl Command for BitsAnd { input.map( move |value| binary_op(&value, &target, little_endian, |(l, r)| l & r, head), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-cmd-extra/src/extra/bits/into.rs b/crates/nu-cmd-extra/src/extra/bits/into.rs index cf85f92ac5..9ff8bd0a05 100644 --- a/crates/nu-cmd-extra/src/extra/bits/into.rs +++ b/crates/nu-cmd-extra/src/extra/bits/into.rs @@ -1,6 +1,9 @@ +use std::io::{self, Read, Write}; + use nu_cmd_base::input_handler::{operate, CmdArgument}; use nu_engine::command_prelude::*; +use nu_protocol::Signals; use num_traits::ToPrimitive; pub struct Arguments { @@ -118,12 +121,43 @@ fn into_bits( let cell_paths = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - if let PipelineData::ByteStream(stream, ..) = input { - // TODO: in the future, we may want this to stream out, converting each to bytes - Ok(Value::binary(stream.into_bytes()?, head).into_pipeline_data()) + if let PipelineData::ByteStream(stream, metadata) = input { + Ok(PipelineData::ByteStream( + byte_stream_to_bits(stream, head), + metadata, + )) } else { let args = Arguments { cell_paths }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) + } +} + +fn byte_stream_to_bits(stream: ByteStream, head: Span) -> ByteStream { + if let Some(mut reader) = stream.reader() { + let mut is_first = true; + ByteStream::from_fn( + head, + Signals::empty(), + ByteStreamType::String, + move |buffer| { + let mut byte = [0]; + if reader.read(&mut byte[..]).err_span(head)? > 0 { + // Format the byte as bits + if is_first { + is_first = false; + } else { + buffer.push(b' '); + } + write!(buffer, "{:08b}", byte[0]).expect("format failed"); + Ok(true) + } else { + // EOF + Ok(false) + } + }, + ) + } else { + ByteStream::read(io::empty(), head, Signals::empty(), ByteStreamType::String) } } diff --git a/crates/nu-cmd-extra/src/extra/bits/not.rs b/crates/nu-cmd-extra/src/extra/bits/not.rs index e4c344f137..405cc79d7e 100644 --- a/crates/nu-cmd-extra/src/extra/bits/not.rs +++ b/crates/nu-cmd-extra/src/extra/bits/not.rs @@ -82,7 +82,7 @@ impl Command for BitsNot { number_size, }; - operate(action, args, input, head, engine_state.ctrlc.clone()) + operate(action, args, input, head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-extra/src/extra/bits/or.rs b/crates/nu-cmd-extra/src/extra/bits/or.rs index 2352d65c23..a0af2dc8d0 100644 --- a/crates/nu-cmd-extra/src/extra/bits/or.rs +++ b/crates/nu-cmd-extra/src/extra/bits/or.rs @@ -80,7 +80,7 @@ impl Command for BitsOr { input.map( move |value| binary_op(&value, &target, little_endian, |(l, r)| l | r, head), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-cmd-extra/src/extra/bits/rotate_left.rs b/crates/nu-cmd-extra/src/extra/bits/rotate_left.rs index cbd9d17eb5..5bb9e42f6b 100644 --- a/crates/nu-cmd-extra/src/extra/bits/rotate_left.rs +++ b/crates/nu-cmd-extra/src/extra/bits/rotate_left.rs @@ -86,7 +86,7 @@ impl Command for BitsRol { bits, }; - operate(action, args, input, head, engine_state.ctrlc.clone()) + operate(action, args, input, head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-extra/src/extra/bits/rotate_right.rs b/crates/nu-cmd-extra/src/extra/bits/rotate_right.rs index 0aea603ce1..31e17891a3 100644 --- a/crates/nu-cmd-extra/src/extra/bits/rotate_right.rs +++ b/crates/nu-cmd-extra/src/extra/bits/rotate_right.rs @@ -86,7 +86,7 @@ impl Command for BitsRor { bits, }; - operate(action, args, input, head, engine_state.ctrlc.clone()) + operate(action, args, input, head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-extra/src/extra/bits/shift_left.rs b/crates/nu-cmd-extra/src/extra/bits/shift_left.rs index 049408c24a..6a67a45e0e 100644 --- a/crates/nu-cmd-extra/src/extra/bits/shift_left.rs +++ b/crates/nu-cmd-extra/src/extra/bits/shift_left.rs @@ -88,7 +88,7 @@ impl Command for BitsShl { bits, }; - operate(action, args, input, head, engine_state.ctrlc.clone()) + operate(action, args, input, head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-extra/src/extra/bits/shift_right.rs b/crates/nu-cmd-extra/src/extra/bits/shift_right.rs index d66db68ee5..e45e10ac94 100644 --- a/crates/nu-cmd-extra/src/extra/bits/shift_right.rs +++ b/crates/nu-cmd-extra/src/extra/bits/shift_right.rs @@ -88,7 +88,7 @@ impl Command for BitsShr { bits, }; - operate(action, args, input, head, engine_state.ctrlc.clone()) + operate(action, args, input, head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-extra/src/extra/bits/xor.rs b/crates/nu-cmd-extra/src/extra/bits/xor.rs index 65c3be4e1a..4a71487137 100644 --- a/crates/nu-cmd-extra/src/extra/bits/xor.rs +++ b/crates/nu-cmd-extra/src/extra/bits/xor.rs @@ -80,7 +80,7 @@ impl Command for BitsXor { input.map( move |value| binary_op(&value, &target, little_endian, |(l, r)| l ^ r, head), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-cmd-extra/src/extra/conversions/fmt.rs b/crates/nu-cmd-extra/src/extra/conversions/fmt.rs index fec0745dac..3b682291e7 100644 --- a/crates/nu-cmd-extra/src/extra/conversions/fmt.rs +++ b/crates/nu-cmd-extra/src/extra/conversions/fmt.rs @@ -59,7 +59,7 @@ fn fmt( ) -> Result { let cell_paths: Vec = call.rest(engine_state, stack, 0)?; let args = CellPathOnlyArgs::from(cell_paths); - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) } fn action(input: &Value, _args: &CellPathOnlyArgs, span: Span) -> Value { diff --git a/crates/nu-cmd-extra/src/extra/filters/each_while.rs b/crates/nu-cmd-extra/src/extra/filters/each_while.rs index 58679c8eea..1f30e5b9b9 100644 --- a/crates/nu-cmd-extra/src/extra/filters/each_while.rs +++ b/crates/nu-cmd-extra/src/extra/filters/each_while.rs @@ -89,7 +89,7 @@ impl Command for EachWhile { } }) .fuse() - .into_pipeline_data(head, engine_state.ctrlc.clone())) + .into_pipeline_data(head, engine_state.signals().clone())) } PipelineData::ByteStream(stream, ..) => { let span = stream.span(); @@ -107,7 +107,7 @@ impl Command for EachWhile { } }) .fuse() - .into_pipeline_data(head, engine_state.ctrlc.clone())) + .into_pipeline_data(head, engine_state.signals().clone())) } else { Ok(PipelineData::Empty) } diff --git a/crates/nu-cmd-extra/src/extra/filters/update_cells.rs b/crates/nu-cmd-extra/src/extra/filters/update_cells.rs index c90e933410..b102c13c0d 100644 --- a/crates/nu-cmd-extra/src/extra/filters/update_cells.rs +++ b/crates/nu-cmd-extra/src/extra/filters/update_cells.rs @@ -108,7 +108,7 @@ impl Command for UpdateCells { columns, span: head, } - .into_pipeline_data(head, engine_state.ctrlc.clone()) + .into_pipeline_data(head, engine_state.signals().clone()) .set_metadata(metadata)) } } diff --git a/crates/nu-cmd-extra/src/extra/formats/to/html.rs b/crates/nu-cmd-extra/src/extra/formats/to/html.rs index c2ca80e2c6..83d0b58b3f 100644 --- a/crates/nu-cmd-extra/src/extra/formats/to/html.rs +++ b/crates/nu-cmd-extra/src/extra/formats/to/html.rs @@ -368,6 +368,7 @@ fn theme_demo(span: Span) -> PipelineData { .collect(); Value::list(result, span).into_pipeline_data_with_metadata(PipelineMetadata { data_source: DataSource::HtmlThemes, + content_type: None, }) } diff --git a/crates/nu-cmd-extra/src/extra/math/arccos.rs b/crates/nu-cmd-extra/src/extra/math/arccos.rs index 120fc4df98..5d0a659380 100644 --- a/crates/nu-cmd-extra/src/extra/math/arccos.rs +++ b/crates/nu-cmd-extra/src/extra/math/arccos.rs @@ -45,7 +45,7 @@ impl Command for SubCommand { } input.map( move |value| operate(value, head, use_degrees), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-cmd-extra/src/extra/math/arccosh.rs b/crates/nu-cmd-extra/src/extra/math/arccosh.rs index 30e0d2cfb6..532a388b3f 100644 --- a/crates/nu-cmd-extra/src/extra/math/arccosh.rs +++ b/crates/nu-cmd-extra/src/extra/math/arccosh.rs @@ -41,10 +41,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map( - move |value| operate(value, head), - engine_state.ctrlc.clone(), - ) + input.map(move |value| operate(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-extra/src/extra/math/arcsin.rs b/crates/nu-cmd-extra/src/extra/math/arcsin.rs index a68e0648ef..a85c438367 100644 --- a/crates/nu-cmd-extra/src/extra/math/arcsin.rs +++ b/crates/nu-cmd-extra/src/extra/math/arcsin.rs @@ -45,7 +45,7 @@ impl Command for SubCommand { } input.map( move |value| operate(value, head, use_degrees), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-cmd-extra/src/extra/math/arcsinh.rs b/crates/nu-cmd-extra/src/extra/math/arcsinh.rs index 67addfdba2..91cb814a32 100644 --- a/crates/nu-cmd-extra/src/extra/math/arcsinh.rs +++ b/crates/nu-cmd-extra/src/extra/math/arcsinh.rs @@ -41,10 +41,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map( - move |value| operate(value, head), - engine_state.ctrlc.clone(), - ) + input.map(move |value| operate(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-extra/src/extra/math/arctan.rs b/crates/nu-cmd-extra/src/extra/math/arctan.rs index 9c14203312..f52c8bd40c 100644 --- a/crates/nu-cmd-extra/src/extra/math/arctan.rs +++ b/crates/nu-cmd-extra/src/extra/math/arctan.rs @@ -45,7 +45,7 @@ impl Command for SubCommand { } input.map( move |value| operate(value, head, use_degrees), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-cmd-extra/src/extra/math/arctanh.rs b/crates/nu-cmd-extra/src/extra/math/arctanh.rs index 920e56eeb6..7791b56948 100644 --- a/crates/nu-cmd-extra/src/extra/math/arctanh.rs +++ b/crates/nu-cmd-extra/src/extra/math/arctanh.rs @@ -41,10 +41,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map( - move |value| operate(value, head), - engine_state.ctrlc.clone(), - ) + input.map(move |value| operate(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-extra/src/extra/math/cos.rs b/crates/nu-cmd-extra/src/extra/math/cos.rs index 633c131b8b..252c7dbbd6 100644 --- a/crates/nu-cmd-extra/src/extra/math/cos.rs +++ b/crates/nu-cmd-extra/src/extra/math/cos.rs @@ -44,7 +44,7 @@ impl Command for SubCommand { } input.map( move |value| operate(value, head, use_degrees), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-cmd-extra/src/extra/math/cosh.rs b/crates/nu-cmd-extra/src/extra/math/cosh.rs index a772540b5c..e46d3c4df8 100644 --- a/crates/nu-cmd-extra/src/extra/math/cosh.rs +++ b/crates/nu-cmd-extra/src/extra/math/cosh.rs @@ -41,10 +41,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map( - move |value| operate(value, head), - engine_state.ctrlc.clone(), - ) + input.map(move |value| operate(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-extra/src/extra/math/exp.rs b/crates/nu-cmd-extra/src/extra/math/exp.rs index b89d6f553f..d8f6a52899 100644 --- a/crates/nu-cmd-extra/src/extra/math/exp.rs +++ b/crates/nu-cmd-extra/src/extra/math/exp.rs @@ -41,10 +41,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map( - move |value| operate(value, head), - engine_state.ctrlc.clone(), - ) + input.map(move |value| operate(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-extra/src/extra/math/ln.rs b/crates/nu-cmd-extra/src/extra/math/ln.rs index dd9782b467..694192bc8e 100644 --- a/crates/nu-cmd-extra/src/extra/math/ln.rs +++ b/crates/nu-cmd-extra/src/extra/math/ln.rs @@ -41,10 +41,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map( - move |value| operate(value, head), - engine_state.ctrlc.clone(), - ) + input.map(move |value| operate(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-extra/src/extra/math/sin.rs b/crates/nu-cmd-extra/src/extra/math/sin.rs index 883007d1ed..0caedbabe7 100644 --- a/crates/nu-cmd-extra/src/extra/math/sin.rs +++ b/crates/nu-cmd-extra/src/extra/math/sin.rs @@ -44,7 +44,7 @@ impl Command for SubCommand { } input.map( move |value| operate(value, head, use_degrees), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-cmd-extra/src/extra/math/sinh.rs b/crates/nu-cmd-extra/src/extra/math/sinh.rs index c768dba739..d40db3bcb5 100644 --- a/crates/nu-cmd-extra/src/extra/math/sinh.rs +++ b/crates/nu-cmd-extra/src/extra/math/sinh.rs @@ -41,10 +41,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map( - move |value| operate(value, head), - engine_state.ctrlc.clone(), - ) + input.map(move |value| operate(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-extra/src/extra/math/tan.rs b/crates/nu-cmd-extra/src/extra/math/tan.rs index e10807279d..97c5d2ff93 100644 --- a/crates/nu-cmd-extra/src/extra/math/tan.rs +++ b/crates/nu-cmd-extra/src/extra/math/tan.rs @@ -44,7 +44,7 @@ impl Command for SubCommand { } input.map( move |value| operate(value, head, use_degrees), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-cmd-extra/src/extra/math/tanh.rs b/crates/nu-cmd-extra/src/extra/math/tanh.rs index 4d09f93cf4..6679d04fe5 100644 --- a/crates/nu-cmd-extra/src/extra/math/tanh.rs +++ b/crates/nu-cmd-extra/src/extra/math/tanh.rs @@ -41,10 +41,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map( - move |value| operate(value, head), - engine_state.ctrlc.clone(), - ) + input.map(move |value| operate(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-cmd-extra/src/extra/platform/ansi/gradient.rs b/crates/nu-cmd-extra/src/extra/platform/ansi/gradient.rs index 21c7e42a61..5934b57a5d 100644 --- a/crates/nu-cmd-extra/src/extra/platform/ansi/gradient.rs +++ b/crates/nu-cmd-extra/src/extra/platform/ansi/gradient.rs @@ -140,7 +140,7 @@ fn operate( ret } }, - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-cmd-extra/src/extra/strings/encode_decode/hex.rs b/crates/nu-cmd-extra/src/extra/strings/encode_decode/hex.rs index 7628d6e240..be681b382a 100644 --- a/crates/nu-cmd-extra/src/extra/strings/encode_decode/hex.rs +++ b/crates/nu-cmd-extra/src/extra/strings/encode_decode/hex.rs @@ -88,7 +88,7 @@ pub fn operate( cell_paths, }; - general_operate(action, args, input, call.head, engine_state.ctrlc.clone()) + general_operate(action, args, input, call.head, engine_state.signals()) } fn action( diff --git a/crates/nu-cmd-extra/src/extra/strings/format/command.rs b/crates/nu-cmd-extra/src/extra/strings/format/command.rs index 1c72627779..08df92ef40 100644 --- a/crates/nu-cmd-extra/src/extra/strings/format/command.rs +++ b/crates/nu-cmd-extra/src/extra/strings/format/command.rs @@ -1,5 +1,4 @@ -use nu_engine::{command_prelude::*, get_eval_expression}; -use nu_parser::parse_expression; +use nu_engine::command_prelude::*; use nu_protocol::{ast::PathMember, engine::StateWorkingSet, ListStream}; #[derive(Clone)] @@ -57,14 +56,7 @@ impl Command for FormatPattern { string_span.start + 1, )?; - format( - input_val, - &ops, - engine_state, - &mut working_set, - stack, - call.head, - ) + format(input_val, &ops, engine_state, call.head) } } } @@ -100,8 +92,6 @@ enum FormatOperation { FixedText(String), // raw input is something like {column1.column2} ValueFromColumn(String, Span), - // raw input is something like {$it.column1.column2} or {$var}. - ValueNeedEval(String, Span), } /// Given a pattern that is fed into the Format command, we can process it and subdivide it @@ -110,7 +100,6 @@ enum FormatOperation { /// there without any further processing. /// FormatOperation::ValueFromColumn contains the name of a column whose values will be /// formatted according to the input pattern. -/// FormatOperation::ValueNeedEval contains expression which need to eval, it has the following form: /// "$it.column1.column2" or "$variable" fn extract_formatting_operations( input: String, @@ -161,10 +150,17 @@ fn extract_formatting_operations( if !column_name.is_empty() { if column_need_eval { - output.push(FormatOperation::ValueNeedEval( - column_name.clone(), - Span::new(span_start + column_span_start, span_start + column_span_end), - )); + return Err(ShellError::GenericError { + error: "Removed functionality".into(), + msg: "The ability to use variables ($it) in `format pattern` has been removed." + .into(), + span: Some(error_span), + help: Some( + "You can use other formatting options, such as string interpolation." + .into(), + ), + inner: vec![], + }); } else { output.push(FormatOperation::ValueFromColumn( column_name.clone(), @@ -185,8 +181,6 @@ fn format( input_data: Value, format_operations: &[FormatOperation], engine_state: &EngineState, - working_set: &mut StateWorkingSet, - stack: &mut Stack, head_span: Span, ) -> Result { let data_as_value = input_data; @@ -194,13 +188,7 @@ fn format( // We can only handle a Record or a List of Records match data_as_value { Value::Record { .. } => { - match format_record( - format_operations, - &data_as_value, - engine_state, - working_set, - stack, - ) { + match format_record(format_operations, &data_as_value, engine_state) { Ok(value) => Ok(PipelineData::Value(Value::string(value, head_span), None)), Err(value) => Err(value), } @@ -211,13 +199,7 @@ fn format( for val in vals.iter() { match val { Value::Record { .. } => { - match format_record( - format_operations, - val, - engine_state, - working_set, - stack, - ) { + match format_record(format_operations, val, engine_state) { Ok(value) => { list.push(Value::string(value, head_span)); } @@ -238,7 +220,7 @@ fn format( } } - Ok(ListStream::new(list.into_iter(), head_span, engine_state.ctrlc.clone()).into()) + Ok(ListStream::new(list.into_iter(), head_span, engine_state.signals().clone()).into()) } // Unwrapping this ShellError is a bit unfortunate. // Ideally, its Span would be preserved. @@ -256,12 +238,9 @@ fn format_record( format_operations: &[FormatOperation], data_as_value: &Value, engine_state: &EngineState, - working_set: &mut StateWorkingSet, - stack: &mut Stack, ) -> Result { let config = engine_state.get_config(); let mut output = String::new(); - let eval_expression = get_eval_expression(engine_state); for op in format_operations { match op { @@ -283,23 +262,6 @@ fn format_record( Err(se) => return Err(se), } } - FormatOperation::ValueNeedEval(_col_name, span) => { - let exp = parse_expression(working_set, &[*span]); - match working_set.parse_errors.first() { - None => { - let parsed_result = eval_expression(engine_state, stack, &exp); - if let Ok(val) = parsed_result { - output.push_str(&val.to_abbreviated_string(config)) - } - } - Some(err) => { - return Err(ShellError::TypeMismatch { - err_message: format!("expression is invalid, detail message: {err:?}"), - span: *span, - }) - } - } - } } } Ok(output) diff --git a/crates/nu-cmd-extra/src/extra/strings/str_/case/mod.rs b/crates/nu-cmd-extra/src/extra/strings/str_/case/mod.rs index 980a1b83fc..bfa54921e2 100644 --- a/crates/nu-cmd-extra/src/extra/strings/str_/case/mod.rs +++ b/crates/nu-cmd-extra/src/extra/strings/str_/case/mod.rs @@ -44,7 +44,7 @@ where case_operation, cell_paths, }; - general_operate(action, args, input, call.head, engine_state.ctrlc.clone()) + general_operate(action, args, input, call.head, engine_state.signals()) } fn action(input: &Value, args: &Arguments, head: Span) -> Value diff --git a/crates/nu-cmd-extra/tests/commands/bits/into.rs b/crates/nu-cmd-extra/tests/commands/bits/into.rs new file mode 100644 index 0000000000..b7e7700583 --- /dev/null +++ b/crates/nu-cmd-extra/tests/commands/bits/into.rs @@ -0,0 +1,13 @@ +use nu_test_support::nu; + +#[test] +fn byte_stream_into_bits() { + let result = nu!("[0x[01] 0x[02 03]] | bytes collect | into bits"); + assert_eq!("00000001 00000010 00000011", result.out); +} + +#[test] +fn byte_stream_into_bits_is_stream() { + let result = nu!("[0x[01] 0x[02 03]] | bytes collect | into bits | describe"); + assert_eq!("string (stream)", result.out); +} diff --git a/crates/nu-cmd-extra/tests/commands/bits/mod.rs b/crates/nu-cmd-extra/tests/commands/bits/mod.rs new file mode 100644 index 0000000000..0d4ee04b0d --- /dev/null +++ b/crates/nu-cmd-extra/tests/commands/bits/mod.rs @@ -0,0 +1 @@ +mod into; diff --git a/crates/nu-cmd-extra/tests/commands/mod.rs b/crates/nu-cmd-extra/tests/commands/mod.rs index 2354122e35..fd216cecb6 100644 --- a/crates/nu-cmd-extra/tests/commands/mod.rs +++ b/crates/nu-cmd-extra/tests/commands/mod.rs @@ -1 +1,2 @@ +mod bits; mod bytes; diff --git a/crates/nu-cmd-lang/Cargo.toml b/crates/nu-cmd-lang/Cargo.toml index 7592d4303b..16ac1b893a 100644 --- a/crates/nu-cmd-lang/Cargo.toml +++ b/crates/nu-cmd-lang/Cargo.toml @@ -6,28 +6,26 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-lang" edition = "2021" license = "MIT" name = "nu-cmd-lang" -version = "0.93.1" +version = "0.95.1" [lib] bench = false [dependencies] -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } -nu-utils = { path = "../nu-utils", version = "0.93.1" } +nu-engine = { path = "../nu-engine", version = "0.95.1" } +nu-parser = { path = "../nu-parser", version = "0.95.1" } +nu-protocol = { path = "../nu-protocol", version = "0.95.1" } +nu-utils = { path = "../nu-utils", version = "0.95.1" } itertools = { workspace = true } -shadow-rs = { version = "0.27", default-features = false } +shadow-rs = { version = "0.29", default-features = false } [build-dependencies] -shadow-rs = { version = "0.27", default-features = false } +shadow-rs = { version = "0.29", default-features = false } [features] mimalloc = [] -which-support = [] trash-support = [] sqlite = [] -dataframe = [] static-link-openssl = [] -system-clipboard = [] +system-clipboard = [] \ No newline at end of file diff --git a/crates/nu-cmd-lang/README.md b/crates/nu-cmd-lang/README.md index 836fabf6f8..6c7298c17a 100644 --- a/crates/nu-cmd-lang/README.md +++ b/crates/nu-cmd-lang/README.md @@ -8,7 +8,6 @@ top of including: * nu-command * nu-cli -* nu-cmd-dataframe * nu-cmd-extra As time goes on and the nu language develops further in parallel with nushell we will be adding other command crates to the system. diff --git a/crates/nu-cmd-lang/src/core_commands/break_.rs b/crates/nu-cmd-lang/src/core_commands/break_.rs index 4698f12c34..90cc1a73f2 100644 --- a/crates/nu-cmd-lang/src/core_commands/break_.rs +++ b/crates/nu-cmd-lang/src/core_commands/break_.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Break; @@ -18,6 +19,15 @@ impl Command for Break { .category(Category::Core) } + fn extra_usage(&self) -> &str { + r#"This command is a parser keyword. For details, check: + https://www.nushell.sh/book/thinking_in_nu.html"# + } + + fn command_type(&self) -> CommandType { + CommandType::Keyword + } + fn run( &self, _engine_state: &EngineState, diff --git a/crates/nu-cmd-lang/src/core_commands/collect.rs b/crates/nu-cmd-lang/src/core_commands/collect.rs index 1c28646548..d6282eec35 100644 --- a/crates/nu-cmd-lang/src/core_commands/collect.rs +++ b/crates/nu-cmd-lang/src/core_commands/collect.rs @@ -50,6 +50,7 @@ is particularly large, this can cause high memory usage."# // check where some input came from. Some(PipelineMetadata { data_source: DataSource::FilePath(_), + content_type: None, }) => None, other => other, }; diff --git a/crates/nu-cmd-lang/src/core_commands/const_.rs b/crates/nu-cmd-lang/src/core_commands/const_.rs index f780c5ada9..5b3d03443a 100644 --- a/crates/nu-cmd-lang/src/core_commands/const_.rs +++ b/crates/nu-cmd-lang/src/core_commands/const_.rs @@ -46,6 +46,9 @@ impl Command for Const { call: &Call, _input: PipelineData, ) -> Result { + // This is compiled specially by the IR compiler. The code here is never used when + // running in IR mode. + let call = call.assert_ast_call()?; let var_id = if let Some(id) = call.positional_nth(0).and_then(|pos| pos.as_var()) { id } else { diff --git a/crates/nu-cmd-lang/src/core_commands/continue_.rs b/crates/nu-cmd-lang/src/core_commands/continue_.rs index f65a983269..cfa3e38335 100644 --- a/crates/nu-cmd-lang/src/core_commands/continue_.rs +++ b/crates/nu-cmd-lang/src/core_commands/continue_.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Continue; @@ -18,6 +19,14 @@ impl Command for Continue { .category(Category::Core) } + fn extra_usage(&self) -> &str { + r#"This command is a parser keyword. For details, check: + https://www.nushell.sh/book/thinking_in_nu.html"# + } + + fn command_type(&self) -> CommandType { + CommandType::Keyword + } fn run( &self, _engine_state: &EngineState, diff --git a/crates/nu-cmd-lang/src/core_commands/def.rs b/crates/nu-cmd-lang/src/core_commands/def.rs index eb1124da19..913f74803c 100644 --- a/crates/nu-cmd-lang/src/core_commands/def.rs +++ b/crates/nu-cmd-lang/src/core_commands/def.rs @@ -60,10 +60,15 @@ impl Command for Def { example: r#"def --env foo [] { $env.BAR = "BAZ" }; foo; $env.BAR"#, result: Some(Value::test_string("BAZ")), }, + Example { + description: "cd affects the environment, so '--env' is required to change directory from within a command", + example: r#"def --env gohome [] { cd ~ }; gohome; $env.PWD == ('~' | path expand)"#, + result: Some(Value::test_string("true")), + }, Example { description: "Define a custom wrapper for an external command", - example: r#"def --wrapped my-echo [...rest] { echo $rest }; my-echo spam"#, - result: Some(Value::test_list(vec![Value::test_string("spam")])), + example: r#"def --wrapped my-echo [...rest] { ^echo ...$rest }; my-echo -e 'spam\tspam'"#, + result: Some(Value::test_string("spam\tspam")), }, ] } diff --git a/crates/nu-cmd-lang/src/core_commands/describe.rs b/crates/nu-cmd-lang/src/core_commands/describe.rs index 7d6d7f6f83..3d992f3f33 100644 --- a/crates/nu-cmd-lang/src/core_commands/describe.rs +++ b/crates/nu-cmd-lang/src/core_commands/describe.rs @@ -163,6 +163,8 @@ fn run( let description = match input { PipelineData::ByteStream(stream, ..) => { + let type_ = stream.type_().describe(); + let description = if options.detailed { let origin = match stream.source() { ByteStreamSource::Read(_) => "unknown", @@ -172,14 +174,14 @@ fn run( Value::record( record! { - "type" => Value::string("byte stream", head), + "type" => Value::string(type_, head), "origin" => Value::string(origin, head), "metadata" => metadata_to_value(metadata, head), }, head, ) } else { - Value::string("byte stream", head) + Value::string(type_, head) }; if !options.no_collect { diff --git a/crates/nu-cmd-lang/src/core_commands/do_.rs b/crates/nu-cmd-lang/src/core_commands/do_.rs index 5f14e88c07..bf29a2159a 100644 --- a/crates/nu-cmd-lang/src/core_commands/do_.rs +++ b/crates/nu-cmd-lang/src/core_commands/do_.rs @@ -23,11 +23,7 @@ impl Command for Do { fn signature(&self) -> Signature { Signature::build("do") - .required( - "closure", - SyntaxShape::OneOf(vec![SyntaxShape::Closure(None), SyntaxShape::Any]), - "The closure to run.", - ) + .required("closure", SyntaxShape::Closure(None), "The closure to run.") .input_output_types(vec![(Type::Any, Type::Any)]) .switch( "ignore-errors", @@ -85,6 +81,10 @@ impl Command for Do { bind_args_to(&mut callee_stack, &block.signature, rest, head)?; let eval_block_with_early_return = get_eval_block_with_early_return(engine_state); + + // Applies to all block evaluation once set true + callee_stack.use_ir = caller_stack.has_env_var(engine_state, "NU_USE_IR"); + let result = eval_block_with_early_return(engine_state, &mut callee_stack, block, input); if has_env { @@ -229,14 +229,24 @@ impl Command for Do { result: None, }, Example { - description: "Run the closure, with a positional parameter", - example: r#"do {|x| 100 + $x } 77"#, + description: "Run the closure with a positional, type-checked parameter", + example: r#"do {|x:int| 100 + $x } 77"#, result: Some(Value::test_int(177)), }, Example { - description: "Run the closure, with input", - example: r#"77 | do {|x| 100 + $in }"#, - result: None, // TODO: returns 177 + description: "Run the closure with pipeline input", + example: r#"77 | do { 100 + $in }"#, + result: Some(Value::test_int(177)), + }, + Example { + description: "Run the closure with a default parameter value", + example: r#"77 | do {|x=100| $x + $in }"#, + result: Some(Value::test_int(177)), + }, + Example { + description: "Run the closure with two positional parameters", + example: r#"do {|x,y| $x + $y } 77 100"#, + result: Some(Value::test_int(177)), }, Example { description: "Run the closure and keep changes to the environment", @@ -298,9 +308,7 @@ fn bind_args_to( if let Some(rest_positional) = &signature.rest_positional { let mut rest_items = vec![]; - for result in - val_iter.skip(signature.required_positional.len() + signature.optional_positional.len()) - { + for result in val_iter { rest_items.push(result); } diff --git a/crates/nu-cmd-lang/src/core_commands/error_make.rs b/crates/nu-cmd-lang/src/core_commands/error_make.rs index 987e083cbc..07efcd7885 100644 --- a/crates/nu-cmd-lang/src/core_commands/error_make.rs +++ b/crates/nu-cmd-lang/src/core_commands/error_make.rs @@ -56,16 +56,7 @@ impl Command for ErrorMake { Example { description: "Create a simple custom error", example: r#"error make {msg: "my custom error message"}"#, - result: Some(Value::error( - ShellError::GenericError { - error: "my custom error message".into(), - msg: "".into(), - span: None, - help: None, - inner: vec![], - }, - Span::unknown(), - )), + result: None, }, Example { description: "Create a more complex custom error", @@ -82,16 +73,7 @@ impl Command for ErrorMake { } help: "A help string, suggesting a fix to the user" # optional }"#, - result: Some(Value::error( - ShellError::GenericError { - error: "my custom error message".into(), - msg: "my custom label text".into(), - span: Some(Span::new(123, 456)), - help: Some("A help string, suggesting a fix to the user".into()), - inner: vec![], - }, - Span::unknown(), - )), + result: None, }, Example { description: diff --git a/crates/nu-cmd-lang/src/core_commands/for_.rs b/crates/nu-cmd-lang/src/core_commands/for_.rs index 387af45282..36df743e5f 100644 --- a/crates/nu-cmd-lang/src/core_commands/for_.rs +++ b/crates/nu-cmd-lang/src/core_commands/for_.rs @@ -1,5 +1,5 @@ use nu_engine::{command_prelude::*, get_eval_block, get_eval_expression}; -use nu_protocol::engine::CommandType; +use nu_protocol::{engine::CommandType, Signals}; #[derive(Clone)] pub struct For; @@ -28,11 +28,6 @@ impl Command for For { "Range of the loop.", ) .required("block", SyntaxShape::Block, "The block to run.") - .switch( - "numbered", - "return a numbered item ($it.index and $it.item)", - Some('n'), - ) .creates_scope() .category(Category::Core) } @@ -53,6 +48,9 @@ impl Command for For { call: &Call, _input: PipelineData, ) -> Result { + // This is compiled specially by the IR compiler. The code here is never used when + // running in IR mode. + let call = call.assert_ast_call()?; let head = call.head; let var_id = call .positional_nth(0) @@ -77,9 +75,6 @@ impl Command for For { let value = eval_expression(engine_state, stack, keyword_expr)?; - let numbered = call.has_flag(engine_state, stack, "numbered")?; - - let ctrlc = engine_state.ctrlc.clone(); let engine_state = engine_state.clone(); let block = engine_state.get_block(block_id); @@ -88,29 +83,14 @@ impl Command for For { let span = value.span(); match value { Value::List { vals, .. } => { - for (idx, x) in vals.into_iter().enumerate() { - if nu_utils::ctrl_c::was_pressed(&ctrlc) { - break; - } + for x in vals.into_iter() { + engine_state.signals().check(head)?; // with_env() is used here to ensure that each iteration uses // a different set of environment variables. // Hence, a 'cd' in the first loop won't affect the next loop. - stack.add_var( - var_id, - if numbered { - Value::record( - record! { - "index" => Value::int(idx as i64, head), - "item" => x, - }, - head, - ) - } else { - x - }, - ); + stack.add_var(var_id, x); match eval_block(&engine_state, stack, block, PipelineData::empty()) { Err(ShellError::Break { .. }) => { @@ -136,21 +116,9 @@ impl Command for For { } } Value::Range { val, .. } => { - for (idx, x) in val.into_range_iter(span, ctrlc).enumerate() { - stack.add_var( - var_id, - if numbered { - Value::record( - record! { - "index" => Value::int(idx as i64, head), - "item" => x, - }, - head, - ) - } else { - x - }, - ); + for x in val.into_range_iter(span, Signals::empty()) { + engine_state.signals().check(head)?; + stack.add_var(var_id, x); match eval_block(&engine_state, stack, block, PipelineData::empty()) { Err(ShellError::Break { .. }) => { @@ -198,8 +166,7 @@ impl Command for For { }, Example { description: "Number each item and print a message", - example: - "for $it in ['bob' 'fred'] --numbered { print $\"($it.index) is ($it.item)\" }", + example: r#"for $it in (['bob' 'fred'] | enumerate) { print $"($it.index) is ($it.item)" }"#, result: None, }, ] diff --git a/crates/nu-cmd-lang/src/core_commands/if_.rs b/crates/nu-cmd-lang/src/core_commands/if_.rs index 83808c8e06..8667843770 100644 --- a/crates/nu-cmd-lang/src/core_commands/if_.rs +++ b/crates/nu-cmd-lang/src/core_commands/if_.rs @@ -2,7 +2,7 @@ use nu_engine::{ command_prelude::*, get_eval_block, get_eval_expression, get_eval_expression_with_input, }; use nu_protocol::{ - engine::StateWorkingSet, + engine::{CommandType, StateWorkingSet}, eval_const::{eval_const_subexpression, eval_constant, eval_constant_with_input}, }; @@ -41,6 +41,15 @@ impl Command for If { .category(Category::Core) } + fn extra_usage(&self) -> &str { + r#"This command is a parser keyword. For details, check: + https://www.nushell.sh/book/thinking_in_nu.html"# + } + + fn command_type(&self) -> CommandType { + CommandType::Keyword + } + fn is_const(&self) -> bool { true } @@ -51,6 +60,9 @@ impl Command for If { call: &Call, input: PipelineData, ) -> Result { + // This is compiled specially by the IR compiler. The code here is never used when + // running in IR mode. + let call = call.assert_ast_call()?; let cond = call.positional_nth(0).expect("checked through parser"); let then_block = call .positional_nth(1) @@ -90,6 +102,9 @@ impl Command for If { call: &Call, input: PipelineData, ) -> Result { + // This is compiled specially by the IR compiler. The code here is never used when + // running in IR mode. + let call = call.assert_ast_call()?; let cond = call.positional_nth(0).expect("checked through parser"); let then_block = call .positional_nth(1) @@ -122,6 +137,10 @@ impl Command for If { } } + fn search_terms(&self) -> Vec<&str> { + vec!["else", "conditional"] + } + fn examples(&self) -> Vec { vec![ Example { diff --git a/crates/nu-cmd-lang/src/core_commands/let_.rs b/crates/nu-cmd-lang/src/core_commands/let_.rs index f2da628c31..46324ef39e 100644 --- a/crates/nu-cmd-lang/src/core_commands/let_.rs +++ b/crates/nu-cmd-lang/src/core_commands/let_.rs @@ -46,6 +46,9 @@ impl Command for Let { call: &Call, input: PipelineData, ) -> Result { + // This is compiled specially by the IR compiler. The code here is never used when + // running in IR mode. + let call = call.assert_ast_call()?; let var_id = call .positional_nth(0) .expect("checked through parser") diff --git a/crates/nu-cmd-lang/src/core_commands/loop_.rs b/crates/nu-cmd-lang/src/core_commands/loop_.rs index 9b1e36a057..f495c8d3ae 100644 --- a/crates/nu-cmd-lang/src/core_commands/loop_.rs +++ b/crates/nu-cmd-lang/src/core_commands/loop_.rs @@ -1,4 +1,5 @@ use nu_engine::{command_prelude::*, get_eval_block}; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Loop; @@ -20,6 +21,15 @@ impl Command for Loop { .category(Category::Core) } + fn extra_usage(&self) -> &str { + r#"This command is a parser keyword. For details, check: + https://www.nushell.sh/book/thinking_in_nu.html"# + } + + fn command_type(&self) -> CommandType { + CommandType::Keyword + } + fn run( &self, engine_state: &EngineState, @@ -27,6 +37,10 @@ impl Command for Loop { call: &Call, _input: PipelineData, ) -> Result { + // This is compiled specially by the IR compiler. The code here is never used when + // running in IR mode. + let call = call.assert_ast_call()?; + let head = call.head; let block_id = call .positional_nth(0) .expect("checked through parser") @@ -39,9 +53,7 @@ impl Command for Loop { let stack = &mut stack.push_redirection(None, None); loop { - if nu_utils::ctrl_c::was_pressed(&engine_state.ctrlc) { - break; - } + engine_state.signals().check(head)?; match eval_block(engine_state, stack, block, PipelineData::empty()) { Err(ShellError::Break { .. }) => { diff --git a/crates/nu-cmd-lang/src/core_commands/match_.rs b/crates/nu-cmd-lang/src/core_commands/match_.rs index 41b5c24702..c3a3d61216 100644 --- a/crates/nu-cmd-lang/src/core_commands/match_.rs +++ b/crates/nu-cmd-lang/src/core_commands/match_.rs @@ -1,7 +1,7 @@ use nu_engine::{ command_prelude::*, get_eval_block, get_eval_expression, get_eval_expression_with_input, }; -use nu_protocol::engine::Matcher; +use nu_protocol::engine::{CommandType, Matcher}; #[derive(Clone)] pub struct Match; @@ -27,6 +27,15 @@ impl Command for Match { .category(Category::Core) } + fn extra_usage(&self) -> &str { + r#"This command is a parser keyword. For details, check: + https://www.nushell.sh/book/thinking_in_nu.html"# + } + + fn command_type(&self) -> CommandType { + CommandType::Keyword + } + fn run( &self, engine_state: &EngineState, @@ -34,6 +43,9 @@ impl Command for Match { call: &Call, input: PipelineData, ) -> Result { + // This is compiled specially by the IR compiler. The code here is never used when + // running in IR mode. + let call = call.assert_ast_call()?; let value: Value = call.req(engine_state, stack, 0)?; let matches = call .positional_nth(1) diff --git a/crates/nu-cmd-lang/src/core_commands/mut_.rs b/crates/nu-cmd-lang/src/core_commands/mut_.rs index 5db3c929af..b729590027 100644 --- a/crates/nu-cmd-lang/src/core_commands/mut_.rs +++ b/crates/nu-cmd-lang/src/core_commands/mut_.rs @@ -46,6 +46,9 @@ impl Command for Mut { call: &Call, input: PipelineData, ) -> Result { + // This is compiled specially by the IR compiler. The code here is never used when + // running in IR mode. + let call = call.assert_ast_call()?; let var_id = call .positional_nth(0) .expect("checked through parser") diff --git a/crates/nu-cmd-lang/src/core_commands/overlay/use_.rs b/crates/nu-cmd-lang/src/core_commands/overlay/use_.rs index e8b51fb59b..d6d3ae745a 100644 --- a/crates/nu-cmd-lang/src/core_commands/overlay/use_.rs +++ b/crates/nu-cmd-lang/src/core_commands/overlay/use_.rs @@ -65,9 +65,9 @@ impl Command for OverlayUse { name_arg.item = trim_quotes_str(&name_arg.item).to_string(); let maybe_origin_module_id = - if let Some(overlay_expr) = call.get_parser_info("overlay_expr") { + if let Some(overlay_expr) = call.get_parser_info(caller_stack, "overlay_expr") { if let Expr::Overlay(module_id) = &overlay_expr.expr { - module_id + *module_id } else { return Err(ShellError::NushellFailedSpanned { msg: "Not an overlay".to_string(), @@ -110,7 +110,7 @@ impl Command for OverlayUse { // a) adding a new overlay // b) refreshing an active overlay (the origin module changed) - let module = engine_state.get_module(*module_id); + let module = engine_state.get_module(module_id); // Evaluate the export-env block (if any) and keep its environment if let Some(block_id) = module.env_block { @@ -118,7 +118,7 @@ impl Command for OverlayUse { &name_arg.item, engine_state, caller_stack, - get_dirs_var_from_call(call), + get_dirs_var_from_call(caller_stack, call), )?; let block = engine_state.get_block(block_id); diff --git a/crates/nu-cmd-lang/src/core_commands/scope/command.rs b/crates/nu-cmd-lang/src/core_commands/scope/command.rs index 98439226cf..cc52a8a16f 100644 --- a/crates/nu-cmd-lang/src/core_commands/scope/command.rs +++ b/crates/nu-cmd-lang/src/core_commands/scope/command.rs @@ -1,5 +1,4 @@ use nu_engine::{command_prelude::*, get_full_help}; -use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct Scope; @@ -20,10 +19,6 @@ impl Command for Scope { "Commands for getting info about what is in scope." } - fn command_type(&self) -> CommandType { - CommandType::Keyword - } - fn run( &self, engine_state: &EngineState, diff --git a/crates/nu-cmd-lang/src/core_commands/try_.rs b/crates/nu-cmd-lang/src/core_commands/try_.rs index 0b399e368a..2309897a1b 100644 --- a/crates/nu-cmd-lang/src/core_commands/try_.rs +++ b/crates/nu-cmd-lang/src/core_commands/try_.rs @@ -1,5 +1,5 @@ use nu_engine::{command_prelude::*, get_eval_block, EvalBlockFn}; -use nu_protocol::engine::Closure; +use nu_protocol::engine::{Closure, CommandType}; #[derive(Clone)] pub struct Try; @@ -10,7 +10,7 @@ impl Command for Try { } fn usage(&self) -> &str { - "Try to run a block, if it fails optionally run a catch block." + "Try to run a block, if it fails optionally run a catch closure." } fn signature(&self) -> nu_protocol::Signature { @@ -18,7 +18,7 @@ impl Command for Try { .input_output_types(vec![(Type::Any, Type::Any)]) .required("try_block", SyntaxShape::Block, "Block to run.") .optional( - "catch_block", + "catch_closure", SyntaxShape::Keyword( b"catch".to_vec(), Box::new(SyntaxShape::OneOf(vec![ @@ -26,11 +26,20 @@ impl Command for Try { SyntaxShape::Closure(Some(vec![SyntaxShape::Any])), ])), ), - "Block to run if try block fails.", + "Closure to run if try block fails.", ) .category(Category::Core) } + fn extra_usage(&self) -> &str { + r#"This command is a parser keyword. For details, check: + https://www.nushell.sh/book/thinking_in_nu.html"# + } + + fn command_type(&self) -> CommandType { + CommandType::Keyword + } + fn run( &self, engine_state: &EngineState, @@ -38,6 +47,9 @@ impl Command for Try { call: &Call, input: PipelineData, ) -> Result { + // This is compiled specially by the IR compiler. The code here is never used when + // running in IR mode. + let call = call.assert_ast_call()?; let try_block = call .positional_nth(0) .expect("checked through parser") @@ -85,9 +97,14 @@ impl Command for Try { }, Example { description: "Try to run a missing command", - example: "try { asdfasdf } catch { 'missing' } ", + example: "try { asdfasdf } catch { 'missing' }", result: Some(Value::test_string("missing")), }, + Example { + description: "Try to run a missing command and report the message", + example: "try { asdfasdf } catch { |err| $err.msg }", + result: None, + }, ] } } diff --git a/crates/nu-cmd-lang/src/core_commands/use_.rs b/crates/nu-cmd-lang/src/core_commands/use_.rs index b0f3648304..7f544fa5d4 100644 --- a/crates/nu-cmd-lang/src/core_commands/use_.rs +++ b/crates/nu-cmd-lang/src/core_commands/use_.rs @@ -57,7 +57,7 @@ This command is a parser keyword. For details, check: let Some(Expression { expr: Expr::ImportPattern(import_pattern), .. - }) = call.get_parser_info("import_pattern") + }) = call.get_parser_info(caller_stack, "import_pattern") else { return Err(ShellError::GenericError { error: "Unexpected import".into(), @@ -68,6 +68,9 @@ This command is a parser keyword. For details, check: }); }; + // Necessary so that we can modify the stack. + let import_pattern = import_pattern.clone(); + if let Some(module_id) = import_pattern.head.id { // Add constants for var_id in &import_pattern.constants { @@ -99,7 +102,7 @@ This command is a parser keyword. For details, check: &module_arg_str, engine_state, caller_stack, - get_dirs_var_from_call(call), + get_dirs_var_from_call(caller_stack, call), )?; let maybe_parent = maybe_file_path .as_ref() diff --git a/crates/nu-cmd-lang/src/core_commands/version.rs b/crates/nu-cmd-lang/src/core_commands/version.rs index 7aaa72b38a..5491db65fc 100644 --- a/crates/nu-cmd-lang/src/core_commands/version.rs +++ b/crates/nu-cmd-lang/src/core_commands/version.rs @@ -116,11 +116,18 @@ pub fn version(engine_state: &EngineState, span: Span) -> Result>(); record.push( @@ -160,11 +167,6 @@ fn features_enabled() -> Vec { // NOTE: There should be another way to know features on. - #[cfg(feature = "which-support")] - { - names.push("which".to_string()); - } - #[cfg(feature = "trash-support")] { names.push("trash".to_string()); @@ -175,11 +177,6 @@ fn features_enabled() -> Vec { names.push("sqlite".to_string()); } - #[cfg(feature = "dataframe")] - { - names.push("dataframe".to_string()); - } - #[cfg(feature = "static-link-openssl")] { names.push("static-link-openssl".to_string()); diff --git a/crates/nu-cmd-lang/src/core_commands/while_.rs b/crates/nu-cmd-lang/src/core_commands/while_.rs index bf9076aa0c..a67c47fcab 100644 --- a/crates/nu-cmd-lang/src/core_commands/while_.rs +++ b/crates/nu-cmd-lang/src/core_commands/while_.rs @@ -1,4 +1,5 @@ use nu_engine::{command_prelude::*, get_eval_block, get_eval_expression}; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct While; @@ -29,6 +30,15 @@ impl Command for While { vec!["loop"] } + fn extra_usage(&self) -> &str { + r#"This command is a parser keyword. For details, check: + https://www.nushell.sh/book/thinking_in_nu.html"# + } + + fn command_type(&self) -> CommandType { + CommandType::Keyword + } + fn run( &self, engine_state: &EngineState, @@ -36,6 +46,10 @@ impl Command for While { call: &Call, _input: PipelineData, ) -> Result { + // This is compiled specially by the IR compiler. The code here is never used when + // running in IR mode. + let call = call.assert_ast_call()?; + let head = call.head; let cond = call.positional_nth(0).expect("checked through parser"); let block_id = call .positional_nth(1) @@ -49,9 +63,7 @@ impl Command for While { let stack = &mut stack.push_redirection(None, None); loop { - if nu_utils::ctrl_c::was_pressed(&engine_state.ctrlc) { - break; - } + engine_state.signals().check(head)?; let result = eval_expression(engine_state, stack, cond)?; diff --git a/crates/nu-cmd-plugin/Cargo.toml b/crates/nu-cmd-plugin/Cargo.toml index 0d83c3a407..7d26fe5ddd 100644 --- a/crates/nu-cmd-plugin/Cargo.toml +++ b/crates/nu-cmd-plugin/Cargo.toml @@ -5,16 +5,16 @@ edition = "2021" license = "MIT" name = "nu-cmd-plugin" repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-plugin" -version = "0.93.1" +version = "0.95.1" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-path = { path = "../nu-path", version = "0.93.1" } -nu-protocol = { path = "../nu-protocol", version = "0.93.1", features = ["plugin"] } -nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.93.1" } +nu-engine = { path = "../nu-engine", version = "0.95.1" } +nu-path = { path = "../nu-path", version = "0.95.1" } +nu-protocol = { path = "../nu-protocol", version = "0.95.1", features = ["plugin"] } +nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.95.1" } itertools = { workspace = true } -[dev-dependencies] +[dev-dependencies] \ No newline at end of file diff --git a/crates/nu-cmd-plugin/src/commands/mod.rs b/crates/nu-cmd-plugin/src/commands/mod.rs index 3e927747f1..17ff32faa1 100644 --- a/crates/nu-cmd-plugin/src/commands/mod.rs +++ b/crates/nu-cmd-plugin/src/commands/mod.rs @@ -1,5 +1,3 @@ mod plugin; -mod register; pub use plugin::*; -pub use register::Register; diff --git a/crates/nu-cmd-plugin/src/commands/plugin/add.rs b/crates/nu-cmd-plugin/src/commands/plugin/add.rs index 225941db01..14f3541168 100644 --- a/crates/nu-cmd-plugin/src/commands/plugin/add.rs +++ b/crates/nu-cmd-plugin/src/commands/plugin/add.rs @@ -118,11 +118,12 @@ apparent the next time `nu` is next launched with that plugin registry file. }, )); let interface = plugin.clone().get_plugin(Some((engine_state, stack)))?; + let metadata = interface.get_metadata()?; let commands = interface.get_signature()?; modify_plugin_file(engine_state, stack, call.head, custom_path, |contents| { - // Update the file with the received signatures - let item = PluginRegistryItem::new(plugin.identity(), commands); + // Update the file with the received metadata and signatures + let item = PluginRegistryItem::new(plugin.identity(), metadata, commands); contents.upsert_plugin(item); Ok(()) })?; diff --git a/crates/nu-cmd-plugin/src/commands/plugin/list.rs b/crates/nu-cmd-plugin/src/commands/plugin/list.rs index 6b715a0001..030a3341d6 100644 --- a/crates/nu-cmd-plugin/src/commands/plugin/list.rs +++ b/crates/nu-cmd-plugin/src/commands/plugin/list.rs @@ -16,6 +16,7 @@ impl Command for PluginList { Type::Table( [ ("name".into(), Type::String), + ("version".into(), Type::String), ("is_running".into(), Type::Bool), ("pid".into(), Type::Int), ("filename".into(), Type::String), @@ -43,6 +44,7 @@ impl Command for PluginList { description: "List installed plugins.", result: Some(Value::test_list(vec![Value::test_record(record! { "name" => Value::test_string("inc"), + "version" => Value::test_string(env!("CARGO_PKG_VERSION")), "is_running" => Value::test_bool(true), "pid" => Value::test_int(106480), "filename" => if cfg!(windows) { @@ -98,8 +100,15 @@ impl Command for PluginList { .map(|s| Value::string(s.to_string_lossy(), head)) .unwrap_or(Value::nothing(head)); + let metadata = plugin.metadata(); + let version = metadata + .and_then(|m| m.version) + .map(|s| Value::string(s, head)) + .unwrap_or(Value::nothing(head)); + let record = record! { "name" => Value::string(plugin.identity().name(), head), + "version" => version, "is_running" => Value::bool(plugin.is_running(), head), "pid" => pid, "filename" => Value::string(plugin.identity().filename().to_string_lossy(), head), diff --git a/crates/nu-cmd-plugin/src/commands/plugin/stop.rs b/crates/nu-cmd-plugin/src/commands/plugin/stop.rs index d74ee59a3f..f90eac8411 100644 --- a/crates/nu-cmd-plugin/src/commands/plugin/stop.rs +++ b/crates/nu-cmd-plugin/src/commands/plugin/stop.rs @@ -72,7 +72,7 @@ impl Command for PluginStop { error: format!("Failed to stop the `{}` plugin", name.item), msg: "couldn't find a plugin with this name".into(), span: Some(name.span), - help: Some("you may need to `register` the plugin first".into()), + help: Some("you may need to `plugin add` the plugin first".into()), inner: vec![], }) } diff --git a/crates/nu-cmd-plugin/src/commands/register.rs b/crates/nu-cmd-plugin/src/commands/register.rs deleted file mode 100644 index 2c10456db7..0000000000 --- a/crates/nu-cmd-plugin/src/commands/register.rs +++ /dev/null @@ -1,80 +0,0 @@ -use nu_engine::command_prelude::*; -use nu_protocol::engine::CommandType; - -#[derive(Clone)] -pub struct Register; - -impl Command for Register { - fn name(&self) -> &str { - "register" - } - - fn usage(&self) -> &str { - "Register a plugin." - } - - fn signature(&self) -> nu_protocol::Signature { - Signature::build("register") - .input_output_types(vec![(Type::Nothing, Type::Nothing)]) - .required( - "plugin", - SyntaxShape::Filepath, - "Path of executable for plugin.", - ) - .optional( - "signature", - SyntaxShape::Any, - "Block with signature description as json object.", - ) - .named( - "shell", - SyntaxShape::Filepath, - "path of shell used to run plugin (cmd, sh, python, etc)", - Some('s'), - ) - .category(Category::Plugin) - } - - fn extra_usage(&self) -> &str { - r#" -Deprecated in favor of `plugin add` and `plugin use`. - -This command is a parser keyword. For details, check: - https://www.nushell.sh/book/thinking_in_nu.html -"# - .trim() - } - - fn search_terms(&self) -> Vec<&str> { - vec!["add"] - } - - fn command_type(&self) -> CommandType { - CommandType::Keyword - } - - fn run( - &self, - _engine_state: &EngineState, - _stack: &mut Stack, - _call: &Call, - _input: PipelineData, - ) -> Result { - Ok(PipelineData::empty()) - } - - fn examples(&self) -> Vec { - vec![ - Example { - description: "Register `nu_plugin_query` plugin from ~/.cargo/bin/ dir", - example: r#"register ~/.cargo/bin/nu_plugin_query"#, - result: None, - }, - Example { - description: "Register `nu_plugin_query` plugin from `nu -c` (writes/updates $nu.plugin-path)", - example: r#"let plugin = ((which nu).path.0 | path dirname | path join 'nu_plugin_query'); nu -c $'register ($plugin); version'"#, - result: None, - }, - ] - } -} diff --git a/crates/nu-cmd-plugin/src/default_context.rs b/crates/nu-cmd-plugin/src/default_context.rs index 601dd52cfc..1ef6a905dd 100644 --- a/crates/nu-cmd-plugin/src/default_context.rs +++ b/crates/nu-cmd-plugin/src/default_context.rs @@ -18,7 +18,6 @@ pub fn add_plugin_command_context(mut engine_state: EngineState) -> EngineState PluginRm, PluginStop, PluginUse, - Register, ); working_set.render() diff --git a/crates/nu-cmd-plugin/src/util.rs b/crates/nu-cmd-plugin/src/util.rs index 312ab9e81a..7d5282235f 100644 --- a/crates/nu-cmd-plugin/src/util.rs +++ b/crates/nu-cmd-plugin/src/util.rs @@ -31,11 +31,20 @@ pub(crate) fn modify_plugin_file( })? }; + let file_span = custom_path.as_ref().map(|p| p.span).unwrap_or(span); + // Try to read the plugin file if it exists let mut contents = if fs::metadata(&plugin_registry_file_path).is_ok_and(|m| m.len() > 0) { PluginRegistryFile::read_from( - File::open(&plugin_registry_file_path).err_span(span)?, - Some(span), + File::open(&plugin_registry_file_path).map_err(|err| ShellError::IOErrorSpanned { + msg: format!( + "failed to read `{}`: {}", + plugin_registry_file_path.display(), + err + ), + span: file_span, + })?, + Some(file_span), )? } else { PluginRegistryFile::default() @@ -46,7 +55,14 @@ pub(crate) fn modify_plugin_file( // Save the modified file on success contents.write_to( - File::create(&plugin_registry_file_path).err_span(span)?, + File::create(&plugin_registry_file_path).map_err(|err| ShellError::IOErrorSpanned { + msg: format!( + "failed to create `{}`: {}", + plugin_registry_file_path.display(), + err + ), + span: file_span, + })?, Some(span), )?; diff --git a/crates/nu-color-config/Cargo.toml b/crates/nu-color-config/Cargo.toml index f183f5c054..23af09432c 100644 --- a/crates/nu-color-config/Cargo.toml +++ b/crates/nu-color-config/Cargo.toml @@ -5,18 +5,18 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-color-confi edition = "2021" license = "MIT" name = "nu-color-config" -version = "0.93.1" +version = "0.95.1" [lib] bench = false [dependencies] -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-json = { path = "../nu-json", version = "0.93.1" } +nu-protocol = { path = "../nu-protocol", version = "0.95.1" } +nu-engine = { path = "../nu-engine", version = "0.95.1" } +nu-json = { path = "../nu-json", version = "0.95.1" } nu-ansi-term = { workspace = true } serde = { workspace = true, features = ["derive"] } [dev-dependencies] -nu-test-support = { path = "../nu-test-support", version = "0.93.1" } +nu-test-support = { path = "../nu-test-support", version = "0.95.1" } \ No newline at end of file diff --git a/crates/nu-color-config/src/shape_color.rs b/crates/nu-color-config/src/shape_color.rs index 72cc955e0b..8da397e914 100644 --- a/crates/nu-color-config/src/shape_color.rs +++ b/crates/nu-color-config/src/shape_color.rs @@ -20,6 +20,7 @@ pub fn default_shape_color(shape: &str) -> Style { "shape_flag" => Style::new().fg(Color::Blue).bold(), "shape_float" => Style::new().fg(Color::Purple).bold(), "shape_garbage" => Style::new().fg(Color::White).on(Color::Red).bold(), + "shape_glob_interpolation" => Style::new().fg(Color::Cyan).bold(), "shape_globpattern" => Style::new().fg(Color::Cyan).bold(), "shape_int" => Style::new().fg(Color::Purple).bold(), "shape_internalcall" => Style::new().fg(Color::Cyan).bold(), diff --git a/crates/nu-command/Cargo.toml b/crates/nu-command/Cargo.toml index 6010e16285..9c49a69ff5 100644 --- a/crates/nu-command/Cargo.toml +++ b/crates/nu-command/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" license = "MIT" name = "nu-command" repository = "https://github.com/nushell/nushell/tree/main/crates/nu-command" -version = "0.93.1" +version = "0.95.1" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -13,21 +13,21 @@ version = "0.93.1" bench = false [dependencies] -nu-cmd-base = { path = "../nu-cmd-base", version = "0.93.1" } -nu-color-config = { path = "../nu-color-config", version = "0.93.1" } -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-glob = { path = "../nu-glob", version = "0.93.1" } -nu-json = { path = "../nu-json", version = "0.93.1" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-path = { path = "../nu-path", version = "0.93.1" } -nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.93.1" } -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } -nu-system = { path = "../nu-system", version = "0.93.1" } -nu-table = { path = "../nu-table", version = "0.93.1" } -nu-term-grid = { path = "../nu-term-grid", version = "0.93.1" } -nu-utils = { path = "../nu-utils", version = "0.93.1" } +nu-cmd-base = { path = "../nu-cmd-base", version = "0.95.1" } +nu-color-config = { path = "../nu-color-config", version = "0.95.1" } +nu-engine = { path = "../nu-engine", version = "0.95.1" } +nu-glob = { path = "../nu-glob", version = "0.95.1" } +nu-json = { path = "../nu-json", version = "0.95.1" } +nu-parser = { path = "../nu-parser", version = "0.95.1" } +nu-path = { path = "../nu-path", version = "0.95.1" } +nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.95.1" } +nu-protocol = { path = "../nu-protocol", version = "0.95.1" } +nu-system = { path = "../nu-system", version = "0.95.1" } +nu-table = { path = "../nu-table", version = "0.95.1" } +nu-term-grid = { path = "../nu-term-grid", version = "0.95.1" } +nu-utils = { path = "../nu-utils", version = "0.95.1" } nu-ansi-term = { workspace = true } -nuon = { path = "../nuon", version = "0.93.1" } +nuon = { path = "../nuon", version = "0.95.1" } alphanumeric-sort = { workspace = true } base64 = { workspace = true } @@ -42,6 +42,7 @@ chrono-humanize = { workspace = true } chrono-tz = { workspace = true } crossterm = { workspace = true } csv = { workspace = true } +deunicode = { workspace = true } dialoguer = { workspace = true, default-features = false, features = ["fuzzy-select"] } digest = { workspace = true, default-features = false } dtparse = { workspace = true } @@ -86,7 +87,7 @@ sysinfo = { workspace = true } tabled = { workspace = true, features = ["color"], default-features = false } terminal_size = { workspace = true } titlecase = { workspace = true } -toml = { workspace = true } +toml = { workspace = true, features = ["preserve_order"]} unicode-segmentation = { workspace = true } ureq = { workspace = true, default-features = false, features = ["charset", "gzip", "json", "native-tls"] } url = { workspace = true } @@ -99,7 +100,7 @@ uu_whoami = { workspace = true } uuid = { workspace = true, features = ["v4"] } v_htmlescape = { workspace = true } wax = { workspace = true } -which = { workspace = true, optional = true } +which = { workspace = true } unicode-width = { workspace = true } [target.'cfg(windows)'.dependencies] @@ -134,11 +135,10 @@ workspace = true plugin = ["nu-parser/plugin"] sqlite = ["rusqlite"] trash-support = ["trash"] -which-support = ["which"] [dev-dependencies] -nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.93.1" } -nu-test-support = { path = "../nu-test-support", version = "0.93.1" } +nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.95.1" } +nu-test-support = { path = "../nu-test-support", version = "0.95.1" } dirs-next = { workspace = true } mockito = { workspace = true, default-features = false } @@ -146,3 +146,4 @@ quickcheck = { workspace = true } quickcheck_macros = { workspace = true } rstest = { workspace = true, default-features = false } pretty_assertions = { workspace = true } +tempfile = { workspace = true } \ No newline at end of file diff --git a/crates/nu-command/src/bytes/add.rs b/crates/nu-command/src/bytes/add.rs index 8514718cfd..ab31f74b12 100644 --- a/crates/nu-command/src/bytes/add.rs +++ b/crates/nu-command/src/bytes/add.rs @@ -78,7 +78,7 @@ impl Command for BytesAdd { end, cell_paths, }; - operate(add, arg, input, call.head, engine_state.ctrlc.clone()) + operate(add, arg, input, call.head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/bytes/at.rs b/crates/nu-command/src/bytes/at.rs index e90312603f..5e95f4fd62 100644 --- a/crates/nu-command/src/bytes/at.rs +++ b/crates/nu-command/src/bytes/at.rs @@ -83,7 +83,7 @@ impl Command for BytesAt { cell_paths, }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) } fn examples(&self) -> Vec { @@ -128,48 +128,24 @@ fn action(input: &Value, args: &Arguments, head: Span) -> Value { let range = &args.indexes; match input { Value::Binary { val, .. } => { - use std::cmp::{self, Ordering}; let len = val.len() as isize; - let start = if range.0 < 0 { range.0 + len } else { range.0 }; + let end = if range.1 < 0 { range.1 + len } else { range.1 }; - let end = if range.1 < 0 { - cmp::max(range.1 + len, 0) - } else { - range.1 - }; - - if start < len && end >= 0 { - match start.cmp(&end) { - Ordering::Equal => Value::binary(vec![], head), - Ordering::Greater => Value::error( - ShellError::TypeMismatch { - err_message: "End must be greater than or equal to Start".to_string(), - span: head, - }, - head, - ), - Ordering::Less => Value::binary( - if end == isize::MAX { - val.iter() - .skip(start as usize) - .copied() - .collect::>() - } else { - val.iter() - .skip(start as usize) - .take((end - start) as usize) - .copied() - .collect() - }, - head, - ), - } - } else { + if start > end { Value::binary(vec![], head) + } else { + let val_iter = val.iter().skip(start as usize); + Value::binary( + if end == isize::MAX { + val_iter.copied().collect::>() + } else { + val_iter.take((end - start + 1) as usize).copied().collect() + }, + head, + ) } } - Value::Error { .. } => input.clone(), other => Value::error( diff --git a/crates/nu-command/src/bytes/build_.rs b/crates/nu-command/src/bytes/build_.rs index f6b1327621..9a3599a071 100644 --- a/crates/nu-command/src/bytes/build_.rs +++ b/crates/nu-command/src/bytes/build_.rs @@ -49,10 +49,8 @@ impl Command for BytesBuild { _input: PipelineData, ) -> Result { let mut output = vec![]; - for val in call.rest_iter_flattened(0, |expr| { - let eval_expression = get_eval_expression(engine_state); - eval_expression(engine_state, stack, expr) - })? { + let eval_expression = get_eval_expression(engine_state); + for val in call.rest_iter_flattened(engine_state, stack, eval_expression, 0)? { let val_span = val.span(); match val { Value::Binary { mut val, .. } => output.append(&mut val), diff --git a/crates/nu-command/src/bytes/collect.rs b/crates/nu-command/src/bytes/collect.rs index 9cd34496e4..afb70bfb36 100644 --- a/crates/nu-command/src/bytes/collect.rs +++ b/crates/nu-command/src/bytes/collect.rs @@ -1,3 +1,4 @@ +use itertools::Itertools; use nu_engine::command_prelude::*; #[derive(Clone, Copy)] @@ -35,46 +36,38 @@ impl Command for BytesCollect { input: PipelineData, ) -> Result { let separator: Option> = call.opt(engine_state, stack, 0)?; + + let span = call.head; + // input should be a list of binary data. - let mut output_binary = vec![]; - for value in input { - match value { - Value::Binary { mut val, .. } => { - output_binary.append(&mut val); - // manually concat - // TODO: make use of std::slice::Join when it's available in stable. - if let Some(sep) = &separator { - let mut work_sep = sep.clone(); - output_binary.append(&mut work_sep) - } - } - // Explicitly propagate errors instead of dropping them. - Value::Error { error, .. } => return Err(*error), - other => { - return Err(ShellError::OnlySupportsThisInputType { + let metadata = input.metadata(); + let iter = Itertools::intersperse( + input.into_iter_strict(span)?.map(move |value| { + // Everything is wrapped in Some in case there's a separator, so we can flatten + Some(match value { + // Explicitly propagate errors instead of dropping them. + Value::Error { error, .. } => Err(*error), + Value::Binary { val, .. } => Ok(val), + other => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "binary".into(), wrong_type: other.get_type().to_string(), - dst_span: call.head, + dst_span: span, src_span: other.span(), - }); - } - } - } + }), + }) + }), + Ok(separator).transpose(), + ) + .flatten(); - match separator { - None => Ok(Value::binary(output_binary, call.head).into_pipeline_data()), - Some(sep) => { - if output_binary.is_empty() { - Ok(Value::binary(output_binary, call.head).into_pipeline_data()) - } else { - // have push one extra separator in previous step, pop them out. - for _ in sep { - let _ = output_binary.pop(); - } - Ok(Value::binary(output_binary, call.head).into_pipeline_data()) - } - } - } + let output = ByteStream::from_result_iter( + iter, + span, + engine_state.signals().clone(), + ByteStreamType::Binary, + ); + + Ok(PipelineData::ByteStream(output, metadata)) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/bytes/ends_with.rs b/crates/nu-command/src/bytes/ends_with.rs index d6174a189c..8e3966716c 100644 --- a/crates/nu-command/src/bytes/ends_with.rs +++ b/crates/nu-command/src/bytes/ends_with.rs @@ -102,7 +102,7 @@ impl Command for BytesEndsWith { pattern, cell_paths, }; - operate(ends_with, arg, input, head, engine_state.ctrlc.clone()) + operate(ends_with, arg, input, head, engine_state.signals()) } } diff --git a/crates/nu-command/src/bytes/index_of.rs b/crates/nu-command/src/bytes/index_of.rs index bdf51b24d9..e10bd6c200 100644 --- a/crates/nu-command/src/bytes/index_of.rs +++ b/crates/nu-command/src/bytes/index_of.rs @@ -71,7 +71,7 @@ impl Command for BytesIndexOf { all: call.has_flag(engine_state, stack, "all")?, cell_paths, }; - operate(index_of, arg, input, call.head, engine_state.ctrlc.clone()) + operate(index_of, arg, input, call.head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/bytes/length.rs b/crates/nu-command/src/bytes/length.rs index aaaf23e0a5..78b3d31eac 100644 --- a/crates/nu-command/src/bytes/length.rs +++ b/crates/nu-command/src/bytes/length.rs @@ -46,7 +46,7 @@ impl Command for BytesLen { ) -> Result { let cell_paths: Vec = call.rest(engine_state, stack, 1)?; let arg = CellPathOnlyArgs::from(cell_paths); - operate(length, arg, input, call.head, engine_state.ctrlc.clone()) + operate(length, arg, input, call.head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/bytes/remove.rs b/crates/nu-command/src/bytes/remove.rs index 9afef07e8b..34d3c427f4 100644 --- a/crates/nu-command/src/bytes/remove.rs +++ b/crates/nu-command/src/bytes/remove.rs @@ -73,7 +73,7 @@ impl Command for BytesRemove { all: call.has_flag(engine_state, stack, "all")?, }; - operate(remove, arg, input, call.head, engine_state.ctrlc.clone()) + operate(remove, arg, input, call.head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/bytes/replace.rs b/crates/nu-command/src/bytes/replace.rs index ab7ede7588..db2b6fb790 100644 --- a/crates/nu-command/src/bytes/replace.rs +++ b/crates/nu-command/src/bytes/replace.rs @@ -73,7 +73,7 @@ impl Command for BytesReplace { all: call.has_flag(engine_state, stack, "all")?, }; - operate(replace, arg, input, call.head, engine_state.ctrlc.clone()) + operate(replace, arg, input, call.head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/bytes/reverse.rs b/crates/nu-command/src/bytes/reverse.rs index 171add213d..fd769eaa40 100644 --- a/crates/nu-command/src/bytes/reverse.rs +++ b/crates/nu-command/src/bytes/reverse.rs @@ -42,7 +42,7 @@ impl Command for BytesReverse { ) -> Result { let cell_paths: Vec = call.rest(engine_state, stack, 0)?; let arg = CellPathOnlyArgs::from(cell_paths); - operate(reverse, arg, input, call.head, engine_state.ctrlc.clone()) + operate(reverse, arg, input, call.head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/bytes/starts_with.rs b/crates/nu-command/src/bytes/starts_with.rs index 92cc16f02c..89cc4f7afc 100644 --- a/crates/nu-command/src/bytes/starts_with.rs +++ b/crates/nu-command/src/bytes/starts_with.rs @@ -79,7 +79,7 @@ impl Command for BytesStartsWith { pattern, cell_paths, }; - operate(starts_with, arg, input, head, engine_state.ctrlc.clone()) + operate(starts_with, arg, input, head, engine_state.signals()) } } diff --git a/crates/nu-command/src/charting/histogram.rs b/crates/nu-command/src/charting/histogram.rs index 52964b087d..29515c96c5 100755 --- a/crates/nu-command/src/charting/histogram.rs +++ b/crates/nu-command/src/charting/histogram.rs @@ -194,7 +194,7 @@ fn run_histogram( if inputs.is_empty() { return Err(ShellError::CantFindColumn { col_name: col_name.clone(), - span: head_span, + span: Some(head_span), src_span: list_span, }); } diff --git a/crates/nu-command/src/conversions/fill.rs b/crates/nu-command/src/conversions/fill.rs index 6507e4a368..eaf8c05da1 100644 --- a/crates/nu-command/src/conversions/fill.rs +++ b/crates/nu-command/src/conversions/fill.rs @@ -165,7 +165,7 @@ fn fill( cell_paths, }; - operate(action, arg, input, call.head, engine_state.ctrlc.clone()) + operate(action, arg, input, call.head, engine_state.signals()) } fn action(input: &Value, args: &Arguments, span: Span) -> Value { diff --git a/crates/nu-command/src/conversions/into/binary.rs b/crates/nu-command/src/conversions/into/binary.rs index 479b0fc7d7..e5c34ae6bc 100644 --- a/crates/nu-command/src/conversions/into/binary.rs +++ b/crates/nu-command/src/conversions/into/binary.rs @@ -127,15 +127,18 @@ fn into_binary( let cell_paths = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - if let PipelineData::ByteStream(stream, ..) = input { - // TODO: in the future, we may want this to stream out, converting each to bytes - Ok(Value::binary(stream.into_bytes()?, head).into_pipeline_data()) + if let PipelineData::ByteStream(stream, metadata) = input { + // Just set the type - that should be good enough + Ok(PipelineData::ByteStream( + stream.with_type(ByteStreamType::Binary), + metadata, + )) } else { let args = Arguments { cell_paths, compact: call.has_flag(engine_state, stack, "compact")?, }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, head, engine_state.signals()) } } diff --git a/crates/nu-command/src/conversions/into/bool.rs b/crates/nu-command/src/conversions/into/bool.rs index b1d433cb93..0fcd33b4a3 100644 --- a/crates/nu-command/src/conversions/into/bool.rs +++ b/crates/nu-command/src/conversions/into/bool.rs @@ -107,7 +107,7 @@ fn into_bool( ) -> Result { let cell_paths: Vec = call.rest(engine_state, stack, 0)?; let args = CellPathOnlyArgs::from(cell_paths); - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) } fn string_to_boolean(s: &str, span: Span) -> Result { diff --git a/crates/nu-command/src/conversions/into/cell_path.rs b/crates/nu-command/src/conversions/into/cell_path.rs index 6da317abd3..1342fb14c0 100644 --- a/crates/nu-command/src/conversions/into/cell_path.rs +++ b/crates/nu-command/src/conversions/into/cell_path.rs @@ -103,7 +103,7 @@ fn into_cell_path(call: &Call, input: PipelineData) -> Result Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, int".into(), - wrong_type: "byte stream".into(), + wrong_type: stream.type_().describe().into(), dst_span: head, src_span: stream.span(), }), @@ -154,7 +154,7 @@ fn record_to_path_member( let Some(value) = record.get("value") else { return Err(ShellError::CantFindColumn { col_name: "value".into(), - span: val_span, + span: Some(val_span), src_span: span, }); }; diff --git a/crates/nu-command/src/conversions/into/datetime.rs b/crates/nu-command/src/conversions/into/datetime.rs index af10732aea..8dc0340ba1 100644 --- a/crates/nu-command/src/conversions/into/datetime.rs +++ b/crates/nu-command/src/conversions/into/datetime.rs @@ -1,5 +1,5 @@ use crate::{generate_strftime_list, parse_date_from_string}; -use chrono::{DateTime, FixedOffset, Local, NaiveTime, TimeZone, Utc}; +use chrono::{DateTime, FixedOffset, Local, NaiveDateTime, NaiveTime, TimeZone, Utc}; use human_date_parser::{from_human_time, ParseResult}; use nu_cmd_base::input_handler::{operate, CmdArgument}; use nu_engine::command_prelude::*; @@ -141,7 +141,7 @@ impl Command for SubCommand { zone_options, cell_paths, }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) } } @@ -162,24 +162,37 @@ impl Command for SubCommand { }; vec![ Example { - description: "Convert any standard timestamp string to datetime", + description: "Convert timestamp string to datetime with timezone offset", example: "'27.02.2021 1:55 pm +0000' | into datetime", #[allow(clippy::inconsistent_digit_grouping)] result: example_result_1(1614434100_000000000), }, Example { - description: "Convert any standard timestamp string to datetime", + description: "Convert standard timestamp string to datetime with timezone offset", example: "'2021-02-27T13:55:40.2246+00:00' | into datetime", #[allow(clippy::inconsistent_digit_grouping)] result: example_result_1(1614434140_224600000), }, Example { description: - "Convert non-standard timestamp string to datetime using a custom format", + "Convert non-standard timestamp string, with timezone offset, to datetime using a custom format", example: "'20210227_135540+0000' | into datetime --format '%Y%m%d_%H%M%S%z'", #[allow(clippy::inconsistent_digit_grouping)] result: example_result_1(1614434140_000000000), }, + Example { + description: "Convert non-standard timestamp string, without timezone offset, to datetime with custom formatting", + example: "'16.11.1984 8:00 am' | into datetime --format '%d.%m.%Y %H:%M %P'", + #[allow(clippy::inconsistent_digit_grouping)] + result: Some(Value::date( + DateTime::from_naive_utc_and_offset( + NaiveDateTime::parse_from_str("16.11.1984 8:00 am", "%d.%m.%Y %H:%M %P") + .expect("date calculation should not fail in test"), + *Local::now().offset(), + ), + Span::test_data(), + )), + }, Example { description: "Convert nanosecond-precision unix timestamp to a datetime with offset from UTC", @@ -372,10 +385,21 @@ fn action(input: &Value, args: &Arguments, head: Span) -> Value { Some(dt) => match DateTime::parse_from_str(val, &dt.0) { Ok(d) => Value::date ( d, head ), Err(reason) => { - Value::error ( - ShellError::CantConvert { to_type: format!("could not parse as datetime using format '{}'", dt.0), from_type: reason.to_string(), span: head, help: Some("you can use `into datetime` without a format string to enable flexible parsing".to_string()) }, - head, - ) + match NaiveDateTime::parse_from_str(val, &dt.0) { + Ok(d) => Value::date ( + DateTime::from_naive_utc_and_offset( + d, + *Local::now().offset(), + ), + head, + ), + Err(_) => { + Value::error ( + ShellError::CantConvert { to_type: format!("could not parse as datetime using format '{}'", dt.0), from_type: reason.to_string(), span: head, help: Some("you can use `into datetime` without a format string to enable flexible parsing".to_string()) }, + head, + ) + } + } } }, @@ -457,7 +481,7 @@ mod tests { } #[test] - fn takes_a_date_format() { + fn takes_a_date_format_with_timezone() { let date_str = Value::test_string("16.11.1984 8:00 am +0000"); let fmt_options = Some(DatetimeFormat("%d.%m.%Y %H:%M %P %z".to_string())); let args = Arguments { @@ -473,6 +497,26 @@ mod tests { assert_eq!(actual, expected) } + #[test] + fn takes_a_date_format_without_timezone() { + let date_str = Value::test_string("16.11.1984 8:00 am"); + let fmt_options = Some(DatetimeFormat("%d.%m.%Y %H:%M %P".to_string())); + let args = Arguments { + zone_options: None, + format_options: fmt_options, + cell_paths: None, + }; + let actual = action(&date_str, &args, Span::test_data()); + let expected = Value::date( + DateTime::from_naive_utc_and_offset( + NaiveDateTime::parse_from_str("16.11.1984 8:00 am", "%d.%m.%Y %H:%M %P").unwrap(), + *Local::now().offset(), + ), + Span::test_data(), + ); + assert_eq!(actual, expected) + } + #[test] fn takes_iso8601_date_format() { let date_str = Value::test_string("2020-08-04T16:39:18+00:00"); diff --git a/crates/nu-command/src/conversions/into/duration.rs b/crates/nu-command/src/conversions/into/duration.rs index 21494f3bcc..b459dd04b1 100644 --- a/crates/nu-command/src/conversions/into/duration.rs +++ b/crates/nu-command/src/conversions/into/duration.rs @@ -166,7 +166,7 @@ fn into_duration( ret } }, - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-command/src/conversions/into/filesize.rs b/crates/nu-command/src/conversions/into/filesize.rs index b3c1e65a3b..5be167e30c 100644 --- a/crates/nu-command/src/conversions/into/filesize.rs +++ b/crates/nu-command/src/conversions/into/filesize.rs @@ -68,7 +68,7 @@ impl Command for SubCommand { ) -> Result { let cell_paths: Vec = call.rest(engine_state, stack, 0)?; let args = CellPathOnlyArgs::from(cell_paths); - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) } fn examples(&self) -> Vec { @@ -122,7 +122,7 @@ pub fn action(input: &Value, _args: &CellPathOnlyArgs, span: Span) -> Value { Value::Filesize { .. } => input.clone(), Value::Int { val, .. } => Value::filesize(*val, value_span), Value::Float { val, .. } => Value::filesize(*val as i64, value_span), - Value::String { val, .. } => match int_from_string(val, value_span) { + Value::String { val, .. } => match i64_from_string(val, value_span) { Ok(val) => Value::filesize(val, value_span), Err(error) => Value::error(error, value_span), }, @@ -138,7 +138,8 @@ pub fn action(input: &Value, _args: &CellPathOnlyArgs, span: Span) -> Value { ), } } -fn int_from_string(a_string: &str, span: Span) -> Result { + +fn i64_from_string(a_string: &str, span: Span) -> Result { // Get the Locale so we know what the thousands separator is let locale = get_system_locale(); @@ -148,29 +149,46 @@ fn int_from_string(a_string: &str, span: Span) -> Result { let clean_string = no_comma_string.trim(); // Hadle negative file size - if let Some(stripped_string) = clean_string.strip_prefix('-') { - match stripped_string.parse::() { - Ok(n) => Ok(-(n.as_u64() as i64)), - Err(_) => Err(ShellError::CantConvert { - to_type: "int".into(), - from_type: "string".into(), - span, - help: None, - }), + if let Some(stripped_negative_string) = clean_string.strip_prefix('-') { + match stripped_negative_string.parse::() { + Ok(n) => i64_from_byte_size(n, true, span), + Err(_) => Err(string_convert_error(span)), + } + } else if let Some(stripped_positive_string) = clean_string.strip_prefix('+') { + match stripped_positive_string.parse::() { + Ok(n) if stripped_positive_string.starts_with(|c: char| c.is_ascii_digit()) => { + i64_from_byte_size(n, false, span) + } + _ => Err(string_convert_error(span)), } } else { match clean_string.parse::() { - Ok(n) => Ok(n.0 as i64), - Err(_) => Err(ShellError::CantConvert { - to_type: "int".into(), - from_type: "string".into(), - span, - help: None, - }), + Ok(n) => i64_from_byte_size(n, false, span), + Err(_) => Err(string_convert_error(span)), } } } +fn i64_from_byte_size( + byte_size: bytesize::ByteSize, + is_negative: bool, + span: Span, +) -> Result { + match i64::try_from(byte_size.as_u64()) { + Ok(n) => Ok(if is_negative { -n } else { n }), + Err(_) => Err(string_convert_error(span)), + } +} + +fn string_convert_error(span: Span) -> ShellError { + ShellError::CantConvert { + to_type: "filesize".into(), + from_type: "string".into(), + span, + help: None, + } +} + #[cfg(test)] mod test { use super::*; diff --git a/crates/nu-command/src/conversions/into/float.rs b/crates/nu-command/src/conversions/into/float.rs index 9ccd7ea03f..43556bb5ef 100644 --- a/crates/nu-command/src/conversions/into/float.rs +++ b/crates/nu-command/src/conversions/into/float.rs @@ -49,7 +49,7 @@ impl Command for SubCommand { ) -> Result { let cell_paths: Vec = call.rest(engine_state, stack, 0)?; let args = CellPathOnlyArgs::from(cell_paths); - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/conversions/into/glob.rs b/crates/nu-command/src/conversions/into/glob.rs index e5d03093f4..ffc3655330 100644 --- a/crates/nu-command/src/conversions/into/glob.rs +++ b/crates/nu-command/src/conversions/into/glob.rs @@ -87,7 +87,7 @@ fn glob_helper( Ok(Value::glob(stream.into_string()?, false, head).into_pipeline_data()) } else { let args = Arguments { cell_paths }; - operate(action, args, input, head, engine_state.ctrlc.clone()) + operate(action, args, input, head, engine_state.signals()) } } diff --git a/crates/nu-command/src/conversions/into/int.rs b/crates/nu-command/src/conversions/into/int.rs index a3f1c92a4f..d4bfa61639 100644 --- a/crates/nu-command/src/conversions/into/int.rs +++ b/crates/nu-command/src/conversions/into/int.rs @@ -158,7 +158,7 @@ impl Command for SubCommand { signed, cell_paths, }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/conversions/into/record.rs b/crates/nu-command/src/conversions/into/record.rs index e867f06e15..1f332d9c85 100644 --- a/crates/nu-command/src/conversions/into/record.rs +++ b/crates/nu-command/src/conversions/into/record.rs @@ -125,7 +125,7 @@ fn into_record( ), }, Value::Range { val, .. } => Value::record( - val.into_range_iter(span, engine_state.ctrlc.clone()) + val.into_range_iter(span, engine_state.signals().clone()) .enumerate() .map(|(idx, val)| (format!("{idx}"), val)) .collect(), diff --git a/crates/nu-command/src/conversions/into/string.rs b/crates/nu-command/src/conversions/into/string.rs index eda4f7e5a5..c394f9fd21 100644 --- a/crates/nu-command/src/conversions/into/string.rs +++ b/crates/nu-command/src/conversions/into/string.rs @@ -156,9 +156,23 @@ fn string_helper( let cell_paths = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - if let PipelineData::ByteStream(stream, ..) = input { - // TODO: in the future, we may want this to stream out, converting each to bytes - Ok(Value::string(stream.into_string()?, head).into_pipeline_data()) + if let PipelineData::ByteStream(stream, metadata) = input { + // Just set the type - that should be good enough. There is no guarantee that the data + // within a string stream is actually valid UTF-8. But refuse to do it if it was already set + // to binary + if stream.type_().is_string_coercible() { + Ok(PipelineData::ByteStream( + stream.with_type(ByteStreamType::String), + metadata, + )) + } else { + Err(ShellError::CantConvert { + to_type: "string".into(), + from_type: "binary".into(), + span: stream.span(), + help: Some("try using the `decode` command".into()), + }) + } } else { let config = engine_state.get_config().clone(); let args = Arguments { @@ -166,7 +180,7 @@ fn string_helper( cell_paths, config, }; - operate(action, args, input, head, engine_state.ctrlc.clone()) + operate(action, args, input, head, engine_state.signals()) } } diff --git a/crates/nu-command/src/conversions/into/value.rs b/crates/nu-command/src/conversions/into/value.rs index 4bf7e68f53..e8787e75bb 100644 --- a/crates/nu-command/src/conversions/into/value.rs +++ b/crates/nu-command/src/conversions/into/value.rs @@ -57,14 +57,12 @@ impl Command for IntoValue { call: &Call, input: PipelineData, ) -> Result { - let engine_state = engine_state.clone(); let metadata = input.metadata(); - let ctrlc = engine_state.ctrlc.clone(); let span = call.head; - let display_as_filesizes = call.has_flag(&engine_state, stack, "prefer-filesizes")?; + let display_as_filesizes = call.has_flag(engine_state, stack, "prefer-filesizes")?; // the columns to update - let columns: Option = call.get_flag(&engine_state, stack, "columns")?; + let columns: Option = call.get_flag(engine_state, stack, "columns")?; let columns: Option> = match columns { Some(val) => Some( val.into_list()? @@ -81,7 +79,7 @@ impl Command for IntoValue { display_as_filesizes, span, } - .into_pipeline_data(span, ctrlc) + .into_pipeline_data(span, engine_state.signals().clone()) .set_metadata(metadata)) } } diff --git a/crates/nu-command/src/database/commands/into_sqlite.rs b/crates/nu-command/src/database/commands/into_sqlite.rs index f3a2e3622a..88a6114ab3 100644 --- a/crates/nu-command/src/database/commands/into_sqlite.rs +++ b/crates/nu-command/src/database/commands/into_sqlite.rs @@ -2,13 +2,8 @@ use crate::database::values::sqlite::{open_sqlite_db, values_to_sql}; use nu_engine::command_prelude::*; use itertools::Itertools; -use std::{ - path::Path, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, -}; +use nu_protocol::Signals; +use std::path::Path; pub const DEFAULT_TABLE_NAME: &str = "main"; @@ -188,23 +183,18 @@ fn operate( let file_name: Spanned = call.req(engine_state, stack, 0)?; let table_name: Option> = call.get_flag(engine_state, stack, "table-name")?; let table = Table::new(&file_name, table_name)?; - let ctrl_c = engine_state.ctrlc.clone(); - - match action(input, table, span, ctrl_c) { - Ok(val) => Ok(val.into_pipeline_data()), - Err(e) => Err(e), - } + Ok(action(input, table, span, engine_state.signals())?.into_pipeline_data()) } fn action( input: PipelineData, table: Table, span: Span, - ctrl_c: Option>, + signals: &Signals, ) -> Result { match input { PipelineData::ListStream(stream, _) => { - insert_in_transaction(stream.into_iter(), span, table, ctrl_c) + insert_in_transaction(stream.into_iter(), span, table, signals) } PipelineData::Value( Value::List { @@ -212,9 +202,9 @@ fn action( internal_span, }, _, - ) => insert_in_transaction(vals.into_iter(), internal_span, table, ctrl_c), + ) => insert_in_transaction(vals.into_iter(), internal_span, table, signals), PipelineData::Value(val, _) => { - insert_in_transaction(std::iter::once(val), span, table, ctrl_c) + insert_in_transaction(std::iter::once(val), span, table, signals) } _ => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list".into(), @@ -229,7 +219,7 @@ fn insert_in_transaction( stream: impl Iterator, span: Span, mut table: Table, - ctrl_c: Option>, + signals: &Signals, ) -> Result { let mut stream = stream.peekable(); let first_val = match stream.peek() { @@ -251,17 +241,15 @@ fn insert_in_transaction( let tx = table.try_init(&first_val)?; for stream_value in stream { - if let Some(ref ctrlc) = ctrl_c { - if ctrlc.load(Ordering::Relaxed) { - tx.rollback().map_err(|e| ShellError::GenericError { - error: "Failed to rollback SQLite transaction".into(), - msg: e.to_string(), - span: None, - help: None, - inner: Vec::new(), - })?; - return Err(ShellError::InterruptedByUser { span: None }); - } + if let Err(err) = signals.check(span) { + tx.rollback().map_err(|e| ShellError::GenericError { + error: "Failed to rollback SQLite transaction".into(), + msg: e.to_string(), + span: None, + help: None, + inner: Vec::new(), + })?; + return Err(err); } let val = stream_value.as_record()?; diff --git a/crates/nu-command/src/database/values/sqlite.rs b/crates/nu-command/src/database/values/sqlite.rs index 483da7672e..f253ba1cbd 100644 --- a/crates/nu-command/src/database/values/sqlite.rs +++ b/crates/nu-command/src/database/values/sqlite.rs @@ -2,7 +2,7 @@ use super::definitions::{ db_column::DbColumn, db_constraint::DbConstraint, db_foreignkey::DbForeignKey, db_index::DbIndex, db_table::DbTable, }; -use nu_protocol::{CustomValue, PipelineData, Record, ShellError, Span, Spanned, Value}; +use nu_protocol::{CustomValue, PipelineData, Record, ShellError, Signals, Span, Spanned, Value}; use rusqlite::{ types::ValueRef, Connection, DatabaseName, Error as SqliteError, OpenFlags, Row, Statement, ToSql, @@ -12,7 +12,6 @@ use std::{ fs::File, io::Read, path::{Path, PathBuf}, - sync::{atomic::AtomicBool, Arc}, }; const SQLITE_MAGIC_BYTES: &[u8] = "SQLite format 3\0".as_bytes(); @@ -24,25 +23,21 @@ pub struct SQLiteDatabase { // 1) YAGNI, 2) it's not obvious how cloning a connection could work, 3) state // management gets tricky quick. Revisit this approach if we find a compelling use case. pub path: PathBuf, - #[serde(skip)] + #[serde(skip, default = "Signals::empty")] // this understandably can't be serialized. think that's OK, I'm not aware of a // reason why a CustomValue would be serialized outside of a plugin - ctrlc: Option>, + signals: Signals, } impl SQLiteDatabase { - pub fn new(path: &Path, ctrlc: Option>) -> Self { + pub fn new(path: &Path, signals: Signals) -> Self { Self { path: PathBuf::from(path), - ctrlc, + signals, } } - pub fn try_from_path( - path: &Path, - span: Span, - ctrlc: Option>, - ) -> Result { + pub fn try_from_path(path: &Path, span: Span, signals: Signals) -> Result { let mut file = File::open(path).map_err(|e| ShellError::ReadingFile { msg: e.to_string(), span, @@ -56,7 +51,7 @@ impl SQLiteDatabase { }) .and_then(|_| { if buf == SQLITE_MAGIC_BYTES { - Ok(SQLiteDatabase::new(path, ctrlc)) + Ok(SQLiteDatabase::new(path, signals)) } else { Err(ShellError::ReadingFile { msg: "Not a SQLite file".into(), @@ -72,7 +67,7 @@ impl SQLiteDatabase { Value::Custom { val, .. } => match val.as_any().downcast_ref::() { Some(db) => Ok(Self { path: db.path.clone(), - ctrlc: db.ctrlc.clone(), + signals: db.signals.clone(), }), None => Err(ShellError::CantConvert { to_type: "database".into(), @@ -107,16 +102,8 @@ impl SQLiteDatabase { call_span: Span, ) -> Result { let conn = open_sqlite_db(&self.path, call_span)?; - - let stream = run_sql_query(conn, sql, params, self.ctrlc.clone()).map_err(|e| { - ShellError::GenericError { - error: "Failed to query SQLite database".into(), - msg: e.to_string(), - span: Some(sql.span), - help: None, - inner: vec![], - } - })?; + let stream = run_sql_query(conn, sql, params, &self.signals) + .map_err(|e| e.into_shell_error(sql.span, "Failed to query SQLite database"))?; Ok(stream) } @@ -352,12 +339,7 @@ impl SQLiteDatabase { impl CustomValue for SQLiteDatabase { fn clone_value(&self, span: Span) -> Value { - let cloned = SQLiteDatabase { - path: self.path.clone(), - ctrlc: self.ctrlc.clone(), - }; - - Value::custom(Box::new(cloned), span) + Value::custom(Box::new(self.clone()), span) } fn type_name(&self) -> String { @@ -366,13 +348,8 @@ impl CustomValue for SQLiteDatabase { fn to_base_value(&self, span: Span) -> Result { let db = open_sqlite_db(&self.path, span)?; - read_entire_sqlite_db(db, span, self.ctrlc.clone()).map_err(|e| ShellError::GenericError { - error: "Failed to read from SQLite database".into(), - msg: e.to_string(), - span: Some(span), - help: None, - inner: vec![], - }) + read_entire_sqlite_db(db, span, &self.signals) + .map_err(|e| e.into_shell_error(span, "Failed to read from SQLite database")) } fn as_any(&self) -> &dyn std::any::Any { @@ -396,20 +373,12 @@ impl CustomValue for SQLiteDatabase { fn follow_path_string( &self, _self_span: Span, - _column_name: String, + column_name: String, path_span: Span, ) -> Result { let db = open_sqlite_db(&self.path, path_span)?; - - read_single_table(db, _column_name, path_span, self.ctrlc.clone()).map_err(|e| { - ShellError::GenericError { - error: "Failed to read from SQLite database".into(), - msg: e.to_string(), - span: Some(path_span), - help: None, - inner: vec![], - } - }) + read_single_table(db, column_name, path_span, &self.signals) + .map_err(|e| e.into_shell_error(path_span, "Failed to read from SQLite database")) } fn typetag_name(&self) -> &'static str { @@ -426,12 +395,12 @@ pub fn open_sqlite_db(path: &Path, call_span: Span) -> Result, params: NuSqlParams, - ctrlc: Option>, -) -> Result { + signals: &Signals, +) -> Result { let stmt = conn.prepare(&sql.item)?; - - prepared_statement_to_nu_list(stmt, params, sql.span, ctrlc) + prepared_statement_to_nu_list(stmt, params, sql.span, signals) } // This is taken from to text local_into_string but tweaks it a bit so that certain formatting does not happen @@ -534,23 +502,56 @@ pub fn nu_value_to_params(value: Value) -> Result { } } +#[derive(Debug)] +enum SqliteOrShellError { + SqliteError(SqliteError), + ShellError(ShellError), +} + +impl From for SqliteOrShellError { + fn from(error: SqliteError) -> Self { + Self::SqliteError(error) + } +} + +impl From for SqliteOrShellError { + fn from(error: ShellError) -> Self { + Self::ShellError(error) + } +} + +impl SqliteOrShellError { + fn into_shell_error(self, span: Span, msg: &str) -> ShellError { + match self { + Self::SqliteError(err) => ShellError::GenericError { + error: msg.into(), + msg: err.to_string(), + span: Some(span), + help: None, + inner: Vec::new(), + }, + Self::ShellError(err) => err, + } + } +} + fn read_single_table( conn: Connection, table_name: String, call_span: Span, - ctrlc: Option>, -) -> Result { + signals: &Signals, +) -> Result { // TODO: Should use params here? let stmt = conn.prepare(&format!("SELECT * FROM [{table_name}]"))?; - prepared_statement_to_nu_list(stmt, NuSqlParams::default(), call_span, ctrlc) + prepared_statement_to_nu_list(stmt, NuSqlParams::default(), call_span, signals) } fn prepared_statement_to_nu_list( mut stmt: Statement, params: NuSqlParams, call_span: Span, - ctrlc: Option>, -) -> Result { + signals: &Signals, +) -> Result { let column_names = stmt .column_names() .into_iter() @@ -576,11 +577,7 @@ fn prepared_statement_to_nu_list( let mut row_values = vec![]; for row_result in row_results { - if nu_utils::ctrl_c::was_pressed(&ctrlc) { - // return whatever we have so far, let the caller decide whether to use it - return Ok(Value::list(row_values, call_span)); - } - + signals.check(call_span)?; if let Ok(row_value) = row_result { row_values.push(row_value); } @@ -606,11 +603,7 @@ fn prepared_statement_to_nu_list( let mut row_values = vec![]; for row_result in row_results { - if nu_utils::ctrl_c::was_pressed(&ctrlc) { - // return whatever we have so far, let the caller decide whether to use it - return Ok(Value::list(row_values, call_span)); - } - + signals.check(call_span)?; if let Ok(row_value) = row_result { row_values.push(row_value); } @@ -626,8 +619,8 @@ fn prepared_statement_to_nu_list( fn read_entire_sqlite_db( conn: Connection, call_span: Span, - ctrlc: Option>, -) -> Result { + signals: &Signals, +) -> Result { let mut tables = Record::new(); let mut get_table_names = @@ -638,12 +631,8 @@ fn read_entire_sqlite_db( let table_name: String = row?; // TODO: Should use params here? let table_stmt = conn.prepare(&format!("select * from [{table_name}]"))?; - let rows = prepared_statement_to_nu_list( - table_stmt, - NuSqlParams::default(), - call_span, - ctrlc.clone(), - )?; + let rows = + prepared_statement_to_nu_list(table_stmt, NuSqlParams::default(), call_span, signals)?; tables.push(table_name, rows); } @@ -710,7 +699,7 @@ mod test { #[test] fn can_read_empty_db() { let db = open_connection_in_memory().unwrap(); - let converted_db = read_entire_sqlite_db(db, Span::test_data(), None).unwrap(); + let converted_db = read_entire_sqlite_db(db, Span::test_data(), &Signals::empty()).unwrap(); let expected = Value::test_record(Record::new()); @@ -730,7 +719,7 @@ mod test { [], ) .unwrap(); - let converted_db = read_entire_sqlite_db(db, Span::test_data(), None).unwrap(); + let converted_db = read_entire_sqlite_db(db, Span::test_data(), &Signals::empty()).unwrap(); let expected = Value::test_record(record! { "person" => Value::test_list(vec![]), @@ -759,7 +748,7 @@ mod test { db.execute("INSERT INTO item (id, name) VALUES (456, 'foo bar')", []) .unwrap(); - let converted_db = read_entire_sqlite_db(db, span, None).unwrap(); + let converted_db = read_entire_sqlite_db(db, span, &Signals::empty()).unwrap(); let expected = Value::test_record(record! { "item" => Value::test_list( diff --git a/crates/nu-command/src/date/humanize.rs b/crates/nu-command/src/date/humanize.rs index 2815571520..d8542a540c 100644 --- a/crates/nu-command/src/date/humanize.rs +++ b/crates/nu-command/src/date/humanize.rs @@ -50,7 +50,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map(move |value| helper(value, head), engine_state.ctrlc.clone()) + input.map(move |value| helper(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/date/list_timezone.rs b/crates/nu-command/src/date/list_timezone.rs index 6f9267947d..56f7fe5376 100644 --- a/crates/nu-command/src/date/list_timezone.rs +++ b/crates/nu-command/src/date/list_timezone.rs @@ -40,7 +40,7 @@ impl Command for SubCommand { head, ) }) - .into_pipeline_data(head, engine_state.ctrlc.clone())) + .into_pipeline_data(head, engine_state.signals().clone())) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/date/to_record.rs b/crates/nu-command/src/date/to_record.rs index f9c0ceff1b..c0b09c040b 100644 --- a/crates/nu-command/src/date/to_record.rs +++ b/crates/nu-command/src/date/to_record.rs @@ -40,7 +40,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map(move |value| helper(value, head), engine_state.ctrlc.clone()) + input.map(move |value| helper(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/date/to_table.rs b/crates/nu-command/src/date/to_table.rs index 36c3f4a94a..7ce8bc171b 100644 --- a/crates/nu-command/src/date/to_table.rs +++ b/crates/nu-command/src/date/to_table.rs @@ -40,7 +40,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map(move |value| helper(value, head), engine_state.ctrlc.clone()) + input.map(move |value| helper(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/date/to_timezone.rs b/crates/nu-command/src/date/to_timezone.rs index 5f41d287ae..3d08d7271b 100644 --- a/crates/nu-command/src/date/to_timezone.rs +++ b/crates/nu-command/src/date/to_timezone.rs @@ -55,7 +55,7 @@ impl Command for SubCommand { } input.map( move |value| helper(value, head, &timezone), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-command/src/debug/debug_.rs b/crates/nu-command/src/debug/debug_.rs index c766081410..f4e5707491 100644 --- a/crates/nu-command/src/debug/debug_.rs +++ b/crates/nu-command/src/debug/debug_.rs @@ -46,7 +46,7 @@ impl Command for Debug { Value::string(x.to_expanded_string(", ", &config), head) } }, - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-command/src/debug/explain.rs b/crates/nu-command/src/debug/explain.rs index b451d6916a..710e37935a 100644 --- a/crates/nu-command/src/debug/explain.rs +++ b/crates/nu-command/src/debug/explain.rs @@ -1,6 +1,6 @@ use nu_engine::{command_prelude::*, get_eval_expression}; use nu_protocol::{ - ast::{Argument, Block, Expr, Expression}, + ast::{self, Argument, Block, Expr, Expression}, engine::Closure, }; @@ -106,7 +106,7 @@ pub fn get_pipeline_elements( fn get_arguments( engine_state: &EngineState, stack: &mut Stack, - call: &Call, + call: &ast::Call, eval_expression_fn: fn(&EngineState, &mut Stack, &Expression) -> Result, ) -> Vec { let mut arg_value = vec![]; diff --git a/crates/nu-command/src/debug/metadata.rs b/crates/nu-command/src/debug/metadata.rs index 135047a3d9..245c150cea 100644 --- a/crates/nu-command/src/debug/metadata.rs +++ b/crates/nu-command/src/debug/metadata.rs @@ -28,6 +28,10 @@ impl Command for Metadata { .category(Category::Debug) } + fn requires_ast_for_arguments(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -35,7 +39,7 @@ impl Command for Metadata { call: &Call, input: PipelineData, ) -> Result { - let arg = call.positional_nth(0); + let arg = call.positional_nth(stack, 0); let head = call.head; match arg { @@ -80,16 +84,23 @@ impl Command for Metadata { match x { PipelineMetadata { data_source: DataSource::Ls, + .. } => record.push("source", Value::string("ls", head)), PipelineMetadata { data_source: DataSource::HtmlThemes, + .. } => record.push("source", Value::string("into html --list", head)), PipelineMetadata { data_source: DataSource::FilePath(path), + .. } => record.push( "source", Value::string(path.to_string_lossy().to_string(), head), ), + _ => {} + } + if let Some(ref content_type) = x.content_type { + record.push("content_type", Value::string(content_type, head)); } } @@ -133,16 +144,23 @@ fn build_metadata_record(arg: &Value, metadata: Option<&PipelineMetadata>, head: match x { PipelineMetadata { data_source: DataSource::Ls, + .. } => record.push("source", Value::string("ls", head)), PipelineMetadata { data_source: DataSource::HtmlThemes, + .. } => record.push("source", Value::string("into html --list", head)), PipelineMetadata { data_source: DataSource::FilePath(path), + .. } => record.push( "source", Value::string(path.to_string_lossy().to_string(), head), ), + _ => {} + } + if let Some(ref content_type) = x.content_type { + record.push("content_type", Value::string(content_type, head)); } } diff --git a/crates/nu-command/src/debug/metadata_set.rs b/crates/nu-command/src/debug/metadata_set.rs index 6608827fda..96f50cfdba 100644 --- a/crates/nu-command/src/debug/metadata_set.rs +++ b/crates/nu-command/src/debug/metadata_set.rs @@ -1,5 +1,5 @@ use nu_engine::command_prelude::*; -use nu_protocol::{DataSource, PipelineMetadata}; +use nu_protocol::DataSource; #[derive(Clone)] pub struct MetadataSet; @@ -27,6 +27,12 @@ impl Command for MetadataSet { "Assign the DataSource::FilePath metadata to the input", Some('f'), ) + .named( + "content-type", + SyntaxShape::String, + "Assign content type metadata to the input", + Some('c'), + ) .allow_variants_without_examples(true) .category(Category::Debug) } @@ -41,33 +47,26 @@ impl Command for MetadataSet { let head = call.head; let ds_fp: Option = call.get_flag(engine_state, stack, "datasource-filepath")?; let ds_ls = call.has_flag(engine_state, stack, "datasource-ls")?; + let content_type: Option = call.get_flag(engine_state, stack, "content-type")?; + let signals = engine_state.signals().clone(); + let metadata = input + .metadata() + .clone() + .unwrap_or_default() + .with_content_type(content_type); match (ds_fp, ds_ls) { - (Some(path), false) => { - let metadata = PipelineMetadata { - data_source: DataSource::FilePath(path.into()), - }; - Ok(input.into_pipeline_data_with_metadata( - head, - engine_state.ctrlc.clone(), - metadata, - )) - } - (None, true) => { - let metadata = PipelineMetadata { - data_source: DataSource::Ls, - }; - Ok(input.into_pipeline_data_with_metadata( - head, - engine_state.ctrlc.clone(), - metadata, - )) - } - _ => Err(ShellError::IncorrectValue { - msg: "Expected either --datasource-ls(-l) or --datasource-filepath(-f)".to_string(), - val_span: head, - call_span: head, - }), + (Some(path), false) => Ok(input.into_pipeline_data_with_metadata( + head, + signals, + metadata.with_data_source(DataSource::FilePath(path.into())), + )), + (None, true) => Ok(input.into_pipeline_data_with_metadata( + head, + signals, + metadata.with_data_source(DataSource::Ls), + )), + _ => Ok(input.into_pipeline_data_with_metadata(head, signals, metadata)), } } @@ -83,18 +82,23 @@ impl Command for MetadataSet { example: "'crates' | metadata set --datasource-filepath $'(pwd)/crates' | metadata", result: None, }, + Example { + description: "Set the metadata of a file path", + example: "'crates' | metadata set --content-type text/plain | metadata", + result: Some(Value::record(record!("content_type" => Value::string("text/plain", Span::test_data())), Span::test_data())), + }, ] } } #[cfg(test)] mod test { + use crate::{test_examples_with_commands, Metadata}; + use super::*; #[test] fn test_examples() { - use crate::test_examples; - - test_examples(MetadataSet {}) + test_examples_with_commands(MetadataSet {}, &[&Metadata {}]) } } diff --git a/crates/nu-command/src/debug/mod.rs b/crates/nu-command/src/debug/mod.rs index f19ddab916..ec18c2be87 100644 --- a/crates/nu-command/src/debug/mod.rs +++ b/crates/nu-command/src/debug/mod.rs @@ -10,6 +10,7 @@ mod profile; mod timeit; mod view; mod view_files; +mod view_ir; mod view_source; mod view_span; @@ -25,5 +26,6 @@ pub use profile::DebugProfile; pub use timeit::TimeIt; pub use view::View; pub use view_files::ViewFiles; +pub use view_ir::ViewIr; pub use view_source::ViewSource; pub use view_span::ViewSpan; diff --git a/crates/nu-command/src/debug/profile.rs b/crates/nu-command/src/debug/profile.rs index bd5de6041a..0cef979094 100644 --- a/crates/nu-command/src/debug/profile.rs +++ b/crates/nu-command/src/debug/profile.rs @@ -28,6 +28,7 @@ impl Command for DebugProfile { Some('v'), ) .switch("expr", "Collect expression types", Some('x')) + .switch("lines", "Collect line numbers", Some('l')) .named( "max-depth", SyntaxShape::Int, @@ -90,6 +91,7 @@ confusing the id/parent_id hierarchy. The --expr flag is helpful for investigati let collect_expanded_source = call.has_flag(engine_state, stack, "expanded-source")?; let collect_values = call.has_flag(engine_state, stack, "values")?; let collect_exprs = call.has_flag(engine_state, stack, "expr")?; + let collect_lines = call.has_flag(engine_state, stack, "lines")?; let max_depth = call .get_flag(engine_state, stack, "max-depth")? .unwrap_or(2); @@ -101,6 +103,7 @@ confusing the id/parent_id hierarchy. The --expr flag is helpful for investigati collect_expanded_source, collect_values, collect_exprs, + collect_lines, call.span(), ); @@ -118,14 +121,11 @@ confusing the id/parent_id hierarchy. The --expr flag is helpful for investigati let result = ClosureEvalOnce::new(engine_state, stack, closure).run_with_input(input); - // TODO: See eval_source() - match result { - Ok(pipeline_data) => { - let _ = pipeline_data.into_value(call.span()); - // pipeline_data.print(engine_state, caller_stack, true, false) - } - Err(_e) => (), // TODO: Report error - } + // Return potential errors + let pipeline_data = result?; + + // Collect the output + let _ = pipeline_data.into_value(call.span()); Ok(engine_state .deactivate_debugger() diff --git a/crates/nu-command/src/debug/timeit.rs b/crates/nu-command/src/debug/timeit.rs index a445679b81..7a48644a6d 100644 --- a/crates/nu-command/src/debug/timeit.rs +++ b/crates/nu-command/src/debug/timeit.rs @@ -32,6 +32,10 @@ impl Command for TimeIt { vec!["timing", "timer", "benchmark", "measure"] } + fn requires_ast_for_arguments(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -39,13 +43,14 @@ impl Command for TimeIt { call: &Call, input: PipelineData, ) -> Result { - let command_to_run = call.positional_nth(0); + // reset outdest, so the command can write to stdout and stderr. + let stack = &mut stack.push_redirection(None, None); + + let command_to_run = call.positional_nth(stack, 0); // Get the start time after all other computation has been done. let start_time = Instant::now(); - // reset outdest, so the command can write to stdout and stderr. - let stack = &mut stack.push_redirection(None, None); if let Some(command_to_run) = command_to_run { if let Some(block_id) = command_to_run.as_block() { let eval_block = get_eval_block(engine_state); @@ -53,7 +58,8 @@ impl Command for TimeIt { eval_block(engine_state, stack, block, input)? } else { let eval_expression_with_input = get_eval_expression_with_input(engine_state); - eval_expression_with_input(engine_state, stack, command_to_run, input)?.0 + let expression = &command_to_run.clone(); + eval_expression_with_input(engine_state, stack, expression, input)?.0 } } else { PipelineData::empty() diff --git a/crates/nu-command/src/debug/view_ir.rs b/crates/nu-command/src/debug/view_ir.rs new file mode 100644 index 0000000000..72136079fd --- /dev/null +++ b/crates/nu-command/src/debug/view_ir.rs @@ -0,0 +1,171 @@ +use nu_engine::command_prelude::*; + +#[derive(Clone)] +pub struct ViewIr; + +impl Command for ViewIr { + fn name(&self) -> &str { + "view ir" + } + + fn signature(&self) -> Signature { + Signature::build(self.name()) + .required( + "target", + SyntaxShape::Any, + "The name or block to view compiled code for.", + ) + .switch( + "json", + "Dump the raw block data as JSON (unstable).", + Some('j'), + ) + .switch( + "decl-id", + "Integer is a declaration ID rather than a block ID.", + Some('d'), + ) + .input_output_type(Type::Nothing, Type::String) + .category(Category::Debug) + } + + fn usage(&self) -> &str { + "View the compiled IR code for a block of code." + } + + fn extra_usage(&self) -> &str { + " +The target can be a closure, the name of a custom command, or an internal block +ID. Closure literals within IR dumps often reference the block by ID (e.g. +`closure(3231)`), so this provides an easy way to read the IR of any embedded +closures. + +The --decl-id option is provided to use a declaration ID instead, which can be +found on `call` instructions. This is sometimes better than using the name, as +the declaration may not be in scope. +" + .trim() + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + _input: PipelineData, + ) -> Result { + let target: Value = call.req(engine_state, stack, 0)?; + let json = call.has_flag(engine_state, stack, "json")?; + let is_decl_id = call.has_flag(engine_state, stack, "decl-id")?; + + let block_id = match target { + Value::Closure { ref val, .. } => val.block_id, + // Decl by name + Value::String { ref val, .. } => { + if let Some(decl_id) = engine_state.find_decl(val.as_bytes(), &[]) { + let decl = engine_state.get_decl(decl_id); + decl.block_id().ok_or_else(|| ShellError::GenericError { + error: format!("Can't view IR for `{val}`"), + msg: "not a custom command".into(), + span: Some(target.span()), + help: Some("internal commands don't have Nushell source code".into()), + inner: vec![], + })? + } else { + return Err(ShellError::GenericError { + error: format!("Can't view IR for `{val}`"), + msg: "can't find a command with this name".into(), + span: Some(target.span()), + help: None, + inner: vec![], + }); + } + } + // Decl by ID - IR dump always shows name of decl, but sometimes it isn't in scope + Value::Int { val, .. } if is_decl_id => { + let decl_id = val + .try_into() + .ok() + .filter(|id| *id < engine_state.num_decls()) + .ok_or_else(|| ShellError::IncorrectValue { + msg: "not a valid decl id".into(), + val_span: target.span(), + call_span: call.head, + })?; + let decl = engine_state.get_decl(decl_id); + decl.block_id().ok_or_else(|| ShellError::GenericError { + error: format!("Can't view IR for `{}`", decl.name()), + msg: "not a custom command".into(), + span: Some(target.span()), + help: Some("internal commands don't have Nushell source code".into()), + inner: vec![], + })? + } + // Block by ID - often shows up in IR + Value::Int { val, .. } => val.try_into().map_err(|_| ShellError::IncorrectValue { + msg: "not a valid block id".into(), + val_span: target.span(), + call_span: call.head, + })?, + // Pass through errors + Value::Error { error, .. } => return Err(*error), + _ => { + return Err(ShellError::TypeMismatch { + err_message: "expected closure, string, or int".into(), + span: call.head, + }) + } + }; + + let Some(block) = engine_state.try_get_block(block_id) else { + return Err(ShellError::GenericError { + error: format!("Unknown block ID: {block_id}"), + msg: "ensure the block ID is correct and try again".into(), + span: Some(target.span()), + help: None, + inner: vec![], + }); + }; + + let ir_block = block + .ir_block + .as_ref() + .ok_or_else(|| ShellError::GenericError { + error: "Can't view IR for this block".into(), + msg: "block is missing compiled representation".into(), + span: block.span, + help: Some("the IrBlock is probably missing due to a compilation error".into()), + inner: vec![], + })?; + + let formatted = if json { + let formatted_instructions = ir_block + .instructions + .iter() + .map(|instruction| { + instruction + .display(engine_state, &ir_block.data) + .to_string() + }) + .collect::>(); + + serde_json::to_string_pretty(&serde_json::json!({ + "block_id": block_id, + "span": block.span, + "ir_block": ir_block, + "formatted_instructions": formatted_instructions, + })) + .map_err(|err| ShellError::GenericError { + error: "JSON serialization failed".into(), + msg: err.to_string(), + span: Some(call.head), + help: None, + inner: vec![], + })? + } else { + format!("{}", ir_block.display(engine_state)) + }; + + Ok(Value::string(formatted, call.head).into_pipeline_data()) + } +} diff --git a/crates/nu-command/src/default_context.rs b/crates/nu-command/src/default_context.rs index ba55472e15..b270a78dce 100644 --- a/crates/nu-command/src/default_context.rs +++ b/crates/nu-command/src/default_context.rs @@ -127,7 +127,7 @@ pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState { SysTemp, SysUsers, UName, - + Which, }; // Help @@ -154,6 +154,7 @@ pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState { TimeIt, View, ViewFiles, + ViewIr, ViewSource, ViewSpan, }; @@ -164,14 +165,14 @@ pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState { #[cfg(any( target_os = "android", target_os = "linux", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", target_os = "macos", target_os = "windows" ))] bind_command! { Ps }; - #[cfg(feature = "which-support")] - bind_command! { Which }; - // Strings bind_command! { Char, @@ -189,6 +190,7 @@ pub fn add_shell_command_context(mut engine_state: EngineState) -> EngineState { Str, StrCapitalize, StrContains, + StrDeunicode, StrDistance, StrDowncase, StrEndswith, diff --git a/crates/nu-command/src/env/config/config_env.rs b/crates/nu-command/src/env/config/config_env.rs index 3c8dff487e..777b6f3aac 100644 --- a/crates/nu-command/src/env/config/config_env.rs +++ b/crates/nu-command/src/env/config/config_env.rs @@ -1,6 +1,7 @@ -use super::utils::gen_command; use nu_cmd_base::util::get_editor; use nu_engine::{command_prelude::*, env_to_strings}; +use nu_protocol::{process::ChildProcess, ByteStream}; +use nu_system::ForegroundChild; #[derive(Clone)] pub struct ConfigEnv; @@ -47,7 +48,7 @@ impl Command for ConfigEnv { engine_state: &EngineState, stack: &mut Stack, call: &Call, - input: PipelineData, + _input: PipelineData, ) -> Result { // `--default` flag handling if call.has_flag(engine_state, stack, "default")? { @@ -55,27 +56,60 @@ impl Command for ConfigEnv { return Ok(Value::string(nu_utils::get_default_env(), head).into_pipeline_data()); } - let env_vars_str = env_to_strings(engine_state, stack)?; - let nu_config = match engine_state.get_config_path("env-path") { - Some(path) => path, - None => { - return Err(ShellError::GenericError { - error: "Could not find $nu.env-path".into(), - msg: "Could not find $nu.env-path".into(), - span: None, - help: None, - inner: vec![], - }); - } + // Find the editor executable. + let (editor_name, editor_args) = get_editor(engine_state, stack, call.head)?; + let paths = nu_engine::env::path_str(engine_state, stack, call.head)?; + let cwd = engine_state.cwd(Some(stack))?; + let editor_executable = crate::which(&editor_name, &paths, cwd.as_ref()).ok_or( + ShellError::ExternalCommand { + label: format!("`{editor_name}` not found"), + help: "Failed to find the editor executable".into(), + span: call.head, + }, + )?; + + let Some(env_path) = engine_state.get_config_path("env-path") else { + return Err(ShellError::GenericError { + error: "Could not find $nu.env-path".into(), + msg: "Could not find $nu.env-path".into(), + span: None, + help: None, + inner: vec![], + }); }; + let env_path = env_path.to_string_lossy().to_string(); - let (item, config_args) = get_editor(engine_state, stack, call.head)?; + // Create the command. + let mut command = std::process::Command::new(editor_executable); - gen_command(call.head, nu_config, item, config_args, env_vars_str).run_with_input( - engine_state, - stack, - input, - true, - ) + // Configure PWD. + command.current_dir(cwd); + + // Configure environment variables. + let envs = env_to_strings(engine_state, stack)?; + command.env_clear(); + command.envs(envs); + + // Configure args. + command.arg(env_path); + command.args(editor_args); + + // Spawn the child process. On Unix, also put the child process to + // foreground if we're in an interactive session. + #[cfg(windows)] + let child = ForegroundChild::spawn(command)?; + #[cfg(unix)] + let child = ForegroundChild::spawn( + command, + engine_state.is_interactive, + &engine_state.pipeline_externals_state, + )?; + + // Wrap the output into a `PipelineData::ByteStream`. + let child = ChildProcess::new(child, None, false, call.head)?; + Ok(PipelineData::ByteStream( + ByteStream::child(child, call.head), + None, + )) } } diff --git a/crates/nu-command/src/env/config/config_nu.rs b/crates/nu-command/src/env/config/config_nu.rs index c43d8d0726..80f0bbc012 100644 --- a/crates/nu-command/src/env/config/config_nu.rs +++ b/crates/nu-command/src/env/config/config_nu.rs @@ -1,6 +1,7 @@ -use super::utils::gen_command; use nu_cmd_base::util::get_editor; use nu_engine::{command_prelude::*, env_to_strings}; +use nu_protocol::{process::ChildProcess, ByteStream}; +use nu_system::ForegroundChild; #[derive(Clone)] pub struct ConfigNu; @@ -51,7 +52,7 @@ impl Command for ConfigNu { engine_state: &EngineState, stack: &mut Stack, call: &Call, - input: PipelineData, + _input: PipelineData, ) -> Result { // `--default` flag handling if call.has_flag(engine_state, stack, "default")? { @@ -59,27 +60,60 @@ impl Command for ConfigNu { return Ok(Value::string(nu_utils::get_default_config(), head).into_pipeline_data()); } - let env_vars_str = env_to_strings(engine_state, stack)?; - let nu_config = match engine_state.get_config_path("config-path") { - Some(path) => path, - None => { - return Err(ShellError::GenericError { - error: "Could not find $nu.config-path".into(), - msg: "Could not find $nu.config-path".into(), - span: None, - help: None, - inner: vec![], - }); - } + // Find the editor executable. + let (editor_name, editor_args) = get_editor(engine_state, stack, call.head)?; + let paths = nu_engine::env::path_str(engine_state, stack, call.head)?; + let cwd = engine_state.cwd(Some(stack))?; + let editor_executable = crate::which(&editor_name, &paths, cwd.as_ref()).ok_or( + ShellError::ExternalCommand { + label: format!("`{editor_name}` not found"), + help: "Failed to find the editor executable".into(), + span: call.head, + }, + )?; + + let Some(config_path) = engine_state.get_config_path("config-path") else { + return Err(ShellError::GenericError { + error: "Could not find $nu.config-path".into(), + msg: "Could not find $nu.config-path".into(), + span: None, + help: None, + inner: vec![], + }); }; + let config_path = config_path.to_string_lossy().to_string(); - let (item, config_args) = get_editor(engine_state, stack, call.head)?; + // Create the command. + let mut command = std::process::Command::new(editor_executable); - gen_command(call.head, nu_config, item, config_args, env_vars_str).run_with_input( - engine_state, - stack, - input, - true, - ) + // Configure PWD. + command.current_dir(cwd); + + // Configure environment variables. + let envs = env_to_strings(engine_state, stack)?; + command.env_clear(); + command.envs(envs); + + // Configure args. + command.arg(config_path); + command.args(editor_args); + + // Spawn the child process. On Unix, also put the child process to + // foreground if we're in an interactive session. + #[cfg(windows)] + let child = ForegroundChild::spawn(command)?; + #[cfg(unix)] + let child = ForegroundChild::spawn( + command, + engine_state.is_interactive, + &engine_state.pipeline_externals_state, + )?; + + // Wrap the output into a `PipelineData::ByteStream`. + let child = ChildProcess::new(child, None, false, call.head)?; + Ok(PipelineData::ByteStream( + ByteStream::child(child, call.head), + None, + )) } } diff --git a/crates/nu-command/src/env/config/mod.rs b/crates/nu-command/src/env/config/mod.rs index 9cd93d2491..fa6a3b4ca3 100644 --- a/crates/nu-command/src/env/config/mod.rs +++ b/crates/nu-command/src/env/config/mod.rs @@ -2,7 +2,6 @@ mod config_; mod config_env; mod config_nu; mod config_reset; -mod utils; pub use config_::ConfigMeta; pub use config_env::ConfigEnv; pub use config_nu::ConfigNu; diff --git a/crates/nu-command/src/env/config/utils.rs b/crates/nu-command/src/env/config/utils.rs deleted file mode 100644 index 4b3ea43483..0000000000 --- a/crates/nu-command/src/env/config/utils.rs +++ /dev/null @@ -1,36 +0,0 @@ -use crate::ExternalCommand; -use nu_protocol::{OutDest, Span, Spanned}; -use std::{collections::HashMap, path::Path}; - -pub(crate) fn gen_command( - span: Span, - config_path: &Path, - item: String, - config_args: Vec, - env_vars_str: HashMap, -) -> ExternalCommand { - let name = Spanned { item, span }; - - let mut args = vec![Spanned { - item: config_path.to_string_lossy().to_string(), - span: Span::unknown(), - }]; - - let number_of_args = config_args.len() + 1; - - for arg in config_args { - args.push(Spanned { - item: arg, - span: Span::unknown(), - }) - } - - ExternalCommand { - name, - args, - arg_keep_raw: vec![false; number_of_args], - out: OutDest::Inherit, - err: OutDest::Inherit, - env_vars: env_vars_str, - } -} diff --git a/crates/nu-command/src/env/export_env.rs b/crates/nu-command/src/env/export_env.rs index 20605a9bb5..7b583c9959 100644 --- a/crates/nu-command/src/env/export_env.rs +++ b/crates/nu-command/src/env/export_env.rs @@ -1,4 +1,5 @@ use nu_engine::{command_prelude::*, get_eval_block, redirect_env}; +use nu_protocol::engine::CommandType; #[derive(Clone)] pub struct ExportEnv; @@ -23,6 +24,19 @@ impl Command for ExportEnv { "Run a block and preserve its environment in a current scope." } + fn extra_usage(&self) -> &str { + r#"This command is a parser keyword. For details, check: + https://www.nushell.sh/book/thinking_in_nu.html"# + } + + fn command_type(&self) -> CommandType { + CommandType::Keyword + } + + fn requires_ast_for_arguments(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -31,7 +45,7 @@ impl Command for ExportEnv { input: PipelineData, ) -> Result { let block_id = call - .positional_nth(0) + .positional_nth(caller_stack, 0) .expect("checked through parser") .as_block() .expect("internal error: missing block"); diff --git a/crates/nu-command/src/env/source_env.rs b/crates/nu-command/src/env/source_env.rs index 71c1e6dc3f..1813a92f1f 100644 --- a/crates/nu-command/src/env/source_env.rs +++ b/crates/nu-command/src/env/source_env.rs @@ -2,6 +2,7 @@ use nu_engine::{ command_prelude::*, find_in_dirs_env, get_dirs_var_from_call, get_eval_block_with_early_return, redirect_env, }; +use nu_protocol::engine::CommandType; use std::path::PathBuf; /// Source a file for environment variables. @@ -28,6 +29,15 @@ impl Command for SourceEnv { "Source the environment from a source file into the current environment." } + fn extra_usage(&self) -> &str { + r#"This command is a parser keyword. For details, check: + https://www.nushell.sh/book/thinking_in_nu.html"# + } + + fn command_type(&self) -> CommandType { + CommandType::Keyword + } + fn run( &self, engine_state: &EngineState, @@ -46,7 +56,7 @@ impl Command for SourceEnv { &source_filename.item, engine_state, caller_stack, - get_dirs_var_from_call(call), + get_dirs_var_from_call(caller_stack, call), )? { PathBuf::from(&path) } else { diff --git a/crates/nu-command/src/env/with_env.rs b/crates/nu-command/src/env/with_env.rs index ff66d21f49..46f7d1eb6e 100644 --- a/crates/nu-command/src/env/with_env.rs +++ b/crates/nu-command/src/env/with_env.rs @@ -1,6 +1,5 @@ use nu_engine::{command_prelude::*, eval_block}; use nu_protocol::{debugger::WithoutDebug, engine::Closure}; -use std::collections::HashMap; #[derive(Clone)] pub struct WithEnv; @@ -58,78 +57,14 @@ fn with_env( call: &Call, input: PipelineData, ) -> Result { - let variable: Value = call.req(engine_state, stack, 0)?; - + let env: Record = call.req(engine_state, stack, 0)?; let capture_block: Closure = call.req(engine_state, stack, 1)?; let block = engine_state.get_block(capture_block.block_id); let mut stack = stack.captures_to_stack_preserve_out_dest(capture_block.captures); - let mut env: HashMap = HashMap::new(); - - match &variable { - Value::List { vals: table, .. } => { - nu_protocol::report_error_new( - engine_state, - &ShellError::GenericError { - error: "Deprecated argument type".into(), - msg: "providing the variables to `with-env` as a list or single row table has been deprecated".into(), - span: Some(variable.span()), - help: Some("use the record form instead".into()), - inner: vec![], - }, - ); - if table.len() == 1 { - // single row([[X W]; [Y Z]]) - match &table[0] { - Value::Record { val, .. } => { - for (k, v) in &**val { - env.insert(k.to_string(), v.clone()); - } - } - x => { - return Err(ShellError::CantConvert { - to_type: "record".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }); - } - } - } else { - // primitive values([X Y W Z]) - for row in table.chunks(2) { - if row.len() == 2 { - env.insert(row[0].coerce_string()?, row[1].clone()); - } - if row.len() == 1 { - return Err(ShellError::IncorrectValue { - msg: format!("Missing value for $env.{}", row[0].coerce_string()?), - val_span: row[0].span(), - call_span: call.head, - }); - } - } - } - } - // when get object by `open x.json` or `from json` - Value::Record { val, .. } => { - for (k, v) in &**val { - env.insert(k.clone(), v.clone()); - } - } - x => { - return Err(ShellError::CantConvert { - to_type: "record".into(), - from_type: x.get_type().to_string(), - span: x.span(), - help: None, - }); - } - }; - // TODO: factor list of prohibited env vars into common place for prohibited in ["PWD", "FILE_PWD", "CURRENT_FILE"] { - if env.contains_key(prohibited) { + if env.contains(prohibited) { return Err(ShellError::AutomaticEnvVarSetManually { envvar_name: prohibited.into(), span: call.head, diff --git a/crates/nu-command/src/filesystem/cd.rs b/crates/nu-command/src/filesystem/cd.rs index 57fe1c17e3..23faf6f67b 100644 --- a/crates/nu-command/src/filesystem/cd.rs +++ b/crates/nu-command/src/filesystem/cd.rs @@ -1,5 +1,6 @@ use nu_cmd_base::util::get_init_cwd; use nu_engine::command_prelude::*; +use nu_path::AbsolutePathBuf; use nu_utils::filesystem::{have_permission, PermissionResult}; #[derive(Clone)] @@ -43,7 +44,10 @@ impl Command for Cd { // If getting PWD failed, default to the initial directory. This way, the // user can use `cd` to recover PWD to a good state. - let cwd = engine_state.cwd(Some(stack)).unwrap_or(get_init_cwd()); + let cwd = engine_state + .cwd(Some(stack)) + .map(AbsolutePathBuf::into_std_path_buf) + .unwrap_or(get_init_cwd()); let path_val = { if let Some(path) = path_val { @@ -135,6 +139,11 @@ impl Command for Cd { example: r#"cd -"#, result: None, }, + Example { + description: "Changing directory with a custom command requires 'def --env'", + example: r#"def --env gohome [] { cd ~ }"#, + result: None, + }, ] } } diff --git a/crates/nu-command/src/filesystem/du.rs b/crates/nu-command/src/filesystem/du.rs index c48a1f7ac1..d34892ace9 100644 --- a/crates/nu-command/src/filesystem/du.rs +++ b/crates/nu-command/src/filesystem/du.rs @@ -3,10 +3,9 @@ use crate::{DirBuilder, DirInfo, FileInfo}; #[allow(deprecated)] use nu_engine::{command_prelude::*, current_dir}; use nu_glob::Pattern; -use nu_protocol::NuGlob; +use nu_protocol::{NuGlob, Signals}; use serde::Deserialize; use std::path::Path; -use std::sync::{atomic::AtomicBool, Arc}; #[derive(Clone)] pub struct Du; @@ -103,7 +102,7 @@ impl Command for Du { let current_dir = current_dir(engine_state, stack)?; let paths = get_rest_for_glob_pattern(engine_state, stack, call, 0)?; - let paths = if call.rest_iter(0).count() == 0 { + let paths = if !call.has_positional_args(stack, 0) { None } else { Some(paths) @@ -120,8 +119,8 @@ impl Command for Du { min_size, }; Ok( - du_for_one_pattern(args, ¤t_dir, tag, engine_state.ctrlc.clone())? - .into_pipeline_data(tag, engine_state.ctrlc.clone()), + du_for_one_pattern(args, ¤t_dir, tag, engine_state.signals())? + .into_pipeline_data(tag, engine_state.signals().clone()), ) } Some(paths) => { @@ -139,7 +138,7 @@ impl Command for Du { args, ¤t_dir, tag, - engine_state.ctrlc.clone(), + engine_state.signals(), )?) } @@ -147,7 +146,7 @@ impl Command for Du { Ok(result_iters .into_iter() .flatten() - .into_pipeline_data(tag, engine_state.ctrlc.clone())) + .into_pipeline_data(tag, engine_state.signals().clone())) } } } @@ -164,8 +163,8 @@ impl Command for Du { fn du_for_one_pattern( args: DuArgs, current_dir: &Path, - call_span: Span, - ctrl_c: Option>, + span: Span, + signals: &Signals, ) -> Result + Send, ShellError> { let exclude = args.exclude.map_or(Ok(None), move |x| { Pattern::new(x.item.as_ref()) @@ -178,7 +177,7 @@ fn du_for_one_pattern( let include_files = args.all; let mut paths = match args.path { - Some(p) => nu_engine::glob_from(&p, current_dir, call_span, None), + Some(p) => nu_engine::glob_from(&p, current_dir, span, None), // The * pattern should never fail. None => nu_engine::glob_from( &Spanned { @@ -186,7 +185,7 @@ fn du_for_one_pattern( span: Span::unknown(), }, current_dir, - call_span, + span, None, ), } @@ -205,7 +204,7 @@ fn du_for_one_pattern( let min_size = args.min_size.map(|f| f.item as u64); let params = DirBuilder { - tag: call_span, + tag: span, min: min_size, deref, exclude, @@ -217,13 +216,13 @@ fn du_for_one_pattern( match p { Ok(a) => { if a.is_dir() { - output.push(DirInfo::new(a, ¶ms, max_depth, ctrl_c.clone()).into()); - } else if let Ok(v) = FileInfo::new(a, deref, call_span) { + output.push(DirInfo::new(a, ¶ms, max_depth, span, signals)?.into()); + } else if let Ok(v) = FileInfo::new(a, deref, span) { output.push(v.into()); } } Err(e) => { - output.push(Value::error(e, call_span)); + output.push(Value::error(e, span)); } } } diff --git a/crates/nu-command/src/filesystem/glob.rs b/crates/nu-command/src/filesystem/glob.rs index b10e8893a0..212c6ceb92 100644 --- a/crates/nu-command/src/filesystem/glob.rs +++ b/crates/nu-command/src/filesystem/glob.rs @@ -1,5 +1,5 @@ use nu_engine::command_prelude::*; -use std::sync::{atomic::AtomicBool, Arc}; +use nu_protocol::Signals; use wax::{Glob as WaxGlob, WalkBehavior, WalkEntry}; #[derive(Clone)] @@ -125,7 +125,6 @@ impl Command for Glob { call: &Call, _input: PipelineData, ) -> Result { - let ctrlc = engine_state.ctrlc.clone(); let span = call.head; let glob_pattern: Spanned = call.req(engine_state, stack, 0)?; let depth = call.get_flag(engine_state, stack, "depth")?; @@ -216,7 +215,14 @@ impl Command for Glob { inner: vec![], })? .flatten(); - glob_to_value(ctrlc, glob_results, no_dirs, no_files, no_symlinks, span) + glob_to_value( + engine_state.signals(), + glob_results, + no_dirs, + no_files, + no_symlinks, + span, + ) } else { let glob_results = glob .walk_with_behavior( @@ -227,12 +233,19 @@ impl Command for Glob { }, ) .flatten(); - glob_to_value(ctrlc, glob_results, no_dirs, no_files, no_symlinks, span) + glob_to_value( + engine_state.signals(), + glob_results, + no_dirs, + no_files, + no_symlinks, + span, + ) }?; Ok(result .into_iter() - .into_pipeline_data(span, engine_state.ctrlc.clone())) + .into_pipeline_data(span, engine_state.signals().clone())) } } @@ -252,7 +265,7 @@ fn convert_patterns(columns: &[Value]) -> Result, ShellError> { } fn glob_to_value<'a>( - ctrlc: Option>, + signals: &Signals, glob_results: impl Iterator>, no_dirs: bool, no_files: bool, @@ -261,10 +274,7 @@ fn glob_to_value<'a>( ) -> Result, ShellError> { let mut result: Vec = Vec::new(); for entry in glob_results { - if nu_utils::ctrl_c::was_pressed(&ctrlc) { - result.clear(); - return Err(ShellError::InterruptedByUser { span: None }); - } + signals.check(span)?; let file_type = entry.file_type(); if !(no_dirs && file_type.is_dir() diff --git a/crates/nu-command/src/filesystem/ls.rs b/crates/nu-command/src/filesystem/ls.rs index a89cf04ed6..807e4f3409 100644 --- a/crates/nu-command/src/filesystem/ls.rs +++ b/crates/nu-command/src/filesystem/ls.rs @@ -6,14 +6,13 @@ use nu_engine::glob_from; use nu_engine::{command_prelude::*, env::current_dir}; use nu_glob::MatchOptions; use nu_path::expand_to_real_path; -use nu_protocol::{DataSource, NuGlob, PipelineMetadata}; +use nu_protocol::{DataSource, NuGlob, PipelineMetadata, Signals}; use pathdiff::diff_paths; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::{ path::PathBuf, - sync::Arc, time::{SystemTime, UNIX_EPOCH}, }; @@ -93,7 +92,6 @@ impl Command for Ls { let du = call.has_flag(engine_state, stack, "du")?; let directory = call.has_flag(engine_state, stack, "directory")?; let use_mime_type = call.has_flag(engine_state, stack, "mime-type")?; - let ctrl_c = engine_state.ctrlc.clone(); let call_span = call.head; #[allow(deprecated)] let cwd = current_dir(engine_state, stack)?; @@ -110,18 +108,19 @@ impl Command for Ls { }; let pattern_arg = get_rest_for_glob_pattern(engine_state, stack, call, 0)?; - let input_pattern_arg = if call.rest_iter(0).count() == 0 { + let input_pattern_arg = if !call.has_positional_args(stack, 0) { None } else { Some(pattern_arg) }; match input_pattern_arg { - None => Ok(ls_for_one_pattern(None, args, ctrl_c.clone(), cwd)? + None => Ok(ls_for_one_pattern(None, args, engine_state.signals(), cwd)? .into_pipeline_data_with_metadata( call_span, - ctrl_c, + engine_state.signals().clone(), PipelineMetadata { data_source: DataSource::Ls, + content_type: None, }, )), Some(pattern) => { @@ -130,7 +129,7 @@ impl Command for Ls { result_iters.push(ls_for_one_pattern( Some(pat), args, - ctrl_c.clone(), + engine_state.signals(), cwd.clone(), )?) } @@ -142,9 +141,10 @@ impl Command for Ls { .flatten() .into_pipeline_data_with_metadata( call_span, - ctrl_c, + engine_state.signals().clone(), PipelineMetadata { data_source: DataSource::Ls, + content_type: None, }, )) } @@ -175,20 +175,32 @@ impl Command for Ls { }, Example { description: "List files and directories whose name do not contain 'bar'", - example: "ls -s | where name !~ bar", + example: "ls | where name !~ bar", result: None, }, Example { - description: "List all dirs in your home directory", + description: "List the full path of all dirs in your home directory", example: "ls -a ~ | where type == dir", result: None, }, Example { description: - "List all dirs in your home directory which have not been modified in 7 days", + "List only the names (not paths) of all dirs in your home directory which have not been modified in 7 days", example: "ls -as ~ | where type == dir and modified < ((date now) - 7day)", result: None, }, + Example { + description: + "Recursively list all files and subdirectories under the current directory using a glob pattern", + example: "ls -a **/*", + result: None, + }, + Example { + description: + "Recursively list *.rs and *.toml files using the glob command", + example: "ls ...(glob **/*.{rs,toml})", + result: None, + }, Example { description: "List given paths and show directories themselves", example: "['/path/to/directory' '/path/to/file'] | each {|| ls -D $in } | flatten", @@ -201,7 +213,7 @@ impl Command for Ls { fn ls_for_one_pattern( pattern_arg: Option>, args: Args, - ctrl_c: Option>, + signals: &Signals, cwd: PathBuf, ) -> Result + Send>, ShellError> { let Args { @@ -328,7 +340,7 @@ fn ls_for_one_pattern( let mut hidden_dirs = vec![]; - let one_ctrl_c = ctrl_c.clone(); + let signals = signals.clone(); Ok(Box::new(paths_peek.filter_map(move |x| match x { Ok(path) => { let metadata = match std::fs::symlink_metadata(&path) { @@ -398,7 +410,7 @@ fn ls_for_one_pattern( call_span, long, du, - one_ctrl_c.clone(), + &signals, use_mime_type, ); match entry { @@ -460,7 +472,6 @@ fn path_contains_hidden_folder(path: &Path, folders: &[PathBuf]) -> bool { #[cfg(unix)] use std::os::unix::fs::FileTypeExt; use std::path::Path; -use std::sync::atomic::AtomicBool; pub fn get_file_type(md: &std::fs::Metadata, display_name: &str, use_mime_type: bool) -> String { let ft = md.file_type(); @@ -509,7 +520,7 @@ pub(crate) fn dir_entry_dict( span: Span, long: bool, du: bool, - ctrl_c: Option>, + signals: &Signals, use_mime_type: bool, ) -> Result { #[cfg(windows)] @@ -604,7 +615,7 @@ pub(crate) fn dir_entry_dict( if md.is_dir() { if du { let params = DirBuilder::new(Span::new(0, 2), None, false, None, false); - let dir_size = DirInfo::new(filename, ¶ms, None, ctrl_c).get_size(); + let dir_size = DirInfo::new(filename, ¶ms, None, span, signals)?.get_size(); Value::filesize(dir_size as i64, span) } else { diff --git a/crates/nu-command/src/filesystem/open.rs b/crates/nu-command/src/filesystem/open.rs index 842eaa5f4c..0351d1d9b2 100644 --- a/crates/nu-command/src/filesystem/open.rs +++ b/crates/nu-command/src/filesystem/open.rs @@ -1,7 +1,7 @@ use super::util::get_rest_for_glob_pattern; #[allow(deprecated)] use nu_engine::{command_prelude::*, current_dir, get_eval_block}; -use nu_protocol::{ByteStream, DataSource, NuGlob, PipelineMetadata}; +use nu_protocol::{ast, ByteStream, DataSource, NuGlob, PipelineMetadata}; use std::path::Path; #[cfg(feature = "sqlite")] @@ -51,13 +51,12 @@ impl Command for Open { ) -> Result { let raw = call.has_flag(engine_state, stack, "raw")?; let call_span = call.head; - let ctrlc = engine_state.ctrlc.clone(); #[allow(deprecated)] let cwd = current_dir(engine_state, stack)?; let mut paths = get_rest_for_glob_pattern(engine_state, stack, call, 0)?; let eval_block = get_eval_block(engine_state); - if paths.is_empty() && call.rest_iter(0).next().is_none() { + if paths.is_empty() && !call.has_positional_args(stack, 0) { // try to use path from pipeline input if there were no positional or spread args let (filename, span) = match input { PipelineData::Value(val, ..) => { @@ -122,8 +121,12 @@ impl Command for Open { } else { #[cfg(feature = "sqlite")] if !raw { - let res = SQLiteDatabase::try_from_path(path, arg_span, ctrlc.clone()) - .map(|db| db.into_value(call.head).into_pipeline_data()); + let res = SQLiteDatabase::try_from_path( + path, + arg_span, + engine_state.signals().clone(), + ) + .map(|db| db.into_value(call.head).into_pipeline_data()); if res.is_ok() { return res; @@ -144,9 +147,10 @@ impl Command for Open { }; let stream = PipelineData::ByteStream( - ByteStream::file(file, call_span, ctrlc.clone()), + ByteStream::file(file, call_span, engine_state.signals().clone()), Some(PipelineMetadata { data_source: DataSource::FilePath(path.to_path_buf()), + content_type: None, }), ); @@ -176,7 +180,8 @@ impl Command for Open { let block = engine_state.get_block(block_id); eval_block(engine_state, stack, block, stream) } else { - decl.run(engine_state, stack, &Call::new(call_span), stream) + let call = ast::Call::new(call_span); + decl.run(engine_state, stack, &(&call).into(), stream) }; output.push(command_output.map_err(|inner| { ShellError::GenericError{ @@ -202,7 +207,7 @@ impl Command for Open { Ok(output .into_iter() .flatten() - .into_pipeline_data(call_span, ctrlc)) + .into_pipeline_data(call_span, engine_state.signals().clone())) } } diff --git a/crates/nu-command/src/filesystem/rm.rs b/crates/nu-command/src/filesystem/rm.rs index 9696ae0c2f..d67046ffe1 100644 --- a/crates/nu-command/src/filesystem/rm.rs +++ b/crates/nu-command/src/filesystem/rm.rs @@ -451,12 +451,7 @@ fn rm( }); for result in iter { - if nu_utils::ctrl_c::was_pressed(&engine_state.ctrlc) { - return Err(ShellError::InterruptedByUser { - span: Some(call.head), - }); - } - + engine_state.signals().check(call.head)?; match result { Ok(None) => {} Ok(Some(msg)) => eprintln!("{msg}"), diff --git a/crates/nu-command/src/filesystem/save.rs b/crates/nu-command/src/filesystem/save.rs index 340ceb4f62..be5073ef20 100644 --- a/crates/nu-command/src/filesystem/save.rs +++ b/crates/nu-command/src/filesystem/save.rs @@ -4,16 +4,13 @@ use nu_engine::get_eval_block; use nu_engine::{command_prelude::*, current_dir}; use nu_path::expand_path_with; use nu_protocol::{ - ast::{Expr, Expression}, - byte_stream::copy_with_interrupt, - process::ChildPipe, - ByteStreamSource, DataSource, OutDest, PipelineMetadata, + ast, byte_stream::copy_with_signals, process::ChildPipe, ByteStreamSource, DataSource, OutDest, + PipelineMetadata, Signals, }; use std::{ fs::File, io::{self, BufRead, BufReader, Read, Write}, path::{Path, PathBuf}, - sync::{atomic::AtomicBool, Arc}, thread, }; @@ -70,24 +67,6 @@ impl Command for Save { let append = call.has_flag(engine_state, stack, "append")?; let force = call.has_flag(engine_state, stack, "force")?; let progress = call.has_flag(engine_state, stack, "progress")?; - let out_append = if let Some(Expression { - expr: Expr::Bool(out_append), - .. - }) = call.get_parser_info("out-append") - { - *out_append - } else { - false - }; - let err_append = if let Some(Expression { - expr: Expr::Bool(err_append), - .. - }) = call.get_parser_info("err-append") - { - *err_append - } else { - false - }; let span = call.head; #[allow(deprecated)] @@ -110,40 +89,33 @@ impl Command for Save { PipelineData::ByteStream(stream, metadata) => { check_saving_to_source_file(metadata.as_ref(), &path, stderr_path.as_ref())?; - let (file, stderr_file) = get_files( - &path, - stderr_path.as_ref(), - append, - out_append, - err_append, - force, - )?; + let (file, stderr_file) = get_files(&path, stderr_path.as_ref(), append, force)?; let size = stream.known_size(); - let ctrlc = engine_state.ctrlc.clone(); + let signals = engine_state.signals(); match stream.into_source() { ByteStreamSource::Read(read) => { - stream_to_file(read, size, ctrlc, file, span, progress)?; + stream_to_file(read, size, signals, file, span, progress)?; } ByteStreamSource::File(source) => { - stream_to_file(source, size, ctrlc, file, span, progress)?; + stream_to_file(source, size, signals, file, span, progress)?; } ByteStreamSource::Child(mut child) => { fn write_or_consume_stderr( stderr: ChildPipe, file: Option, span: Span, - ctrlc: Option>, + signals: &Signals, progress: bool, ) -> Result<(), ShellError> { if let Some(file) = file { match stderr { ChildPipe::Pipe(pipe) => { - stream_to_file(pipe, None, ctrlc, file, span, progress) + stream_to_file(pipe, None, signals, file, span, progress) } ChildPipe::Tee(tee) => { - stream_to_file(tee, None, ctrlc, file, span, progress) + stream_to_file(tee, None, signals, file, span, progress) } }? } else { @@ -163,14 +135,14 @@ impl Command for Save { // delegate a thread to redirect stderr to result. let handler = stderr .map(|stderr| { - let ctrlc = ctrlc.clone(); + let signals = signals.clone(); thread::Builder::new().name("stderr saver".into()).spawn( move || { write_or_consume_stderr( stderr, stderr_file, span, - ctrlc, + &signals, progress, ) }, @@ -181,10 +153,10 @@ impl Command for Save { let res = match stdout { ChildPipe::Pipe(pipe) => { - stream_to_file(pipe, None, ctrlc, file, span, progress) + stream_to_file(pipe, None, signals, file, span, progress) } ChildPipe::Tee(tee) => { - stream_to_file(tee, None, ctrlc, file, span, progress) + stream_to_file(tee, None, signals, file, span, progress) } }; if let Some(h) = handler { @@ -202,7 +174,7 @@ impl Command for Save { stderr, stderr_file, span, - ctrlc, + signals, progress, )?; } @@ -222,14 +194,7 @@ impl Command for Save { stderr_path.as_ref(), )?; - let (mut file, _) = get_files( - &path, - stderr_path.as_ref(), - append, - out_append, - err_append, - force, - )?; + let (mut file, _) = get_files(&path, stderr_path.as_ref(), append, force)?; for val in ls { file.write_all(&value_to_bytes(val)?) .map_err(|err| ShellError::IOError { @@ -259,14 +224,7 @@ impl Command for Save { input_to_bytes(input, Path::new(&path.item), raw, engine_state, stack, span)?; // Only open file after successful conversion - let (mut file, _) = get_files( - &path, - stderr_path.as_ref(), - append, - out_append, - err_append, - force, - )?; + let (mut file, _) = get_files(&path, stderr_path.as_ref(), append, force)?; file.write_all(&bytes).map_err(|err| ShellError::IOError { msg: err.to_string(), @@ -398,7 +356,8 @@ fn convert_to_extension( let eval_block = get_eval_block(engine_state); eval_block(engine_state, stack, block, input) } else { - decl.run(engine_state, stack, &Call::new(span), input) + let call = ast::Call::new(span); + decl.run(engine_state, stack, &(&call).into(), input) } } else { Ok(input) @@ -461,7 +420,7 @@ fn open_file(path: &Path, span: Span, append: bool) -> Result }; file.map_err(|e| ShellError::GenericError { - error: "Permission denied".into(), + error: format!("Problem with [{}], Permission denied", path.display()), msg: e.to_string(), span: Some(span), help: None, @@ -474,19 +433,17 @@ fn get_files( path: &Spanned, stderr_path: Option<&Spanned>, append: bool, - out_append: bool, - err_append: bool, force: bool, ) -> Result<(File, Option), ShellError> { // First check both paths - let (path, path_span) = prepare_path(path, append || out_append, force)?; + let (path, path_span) = prepare_path(path, append, force)?; let stderr_path_and_span = stderr_path .as_ref() - .map(|stderr_path| prepare_path(stderr_path, append || err_append, force)) + .map(|stderr_path| prepare_path(stderr_path, append, force)) .transpose()?; // Only if both files can be used open and possibly truncate them - let file = open_file(path, path_span, append || out_append)?; + let file = open_file(path, path_span, append)?; let stderr_file = stderr_path_and_span .map(|(stderr_path, stderr_path_span)| { @@ -499,7 +456,7 @@ fn get_files( inner: vec![], }) } else { - open_file(stderr_path, stderr_path_span, append || err_append) + open_file(stderr_path, stderr_path_span, append) } }) .transpose()?; @@ -508,9 +465,9 @@ fn get_files( } fn stream_to_file( - mut source: impl Read, + source: impl Read, known_size: Option, - ctrlc: Option>, + signals: &Signals, mut file: File, span: Span, progress: bool, @@ -526,9 +483,9 @@ fn stream_to_file( let mut reader = BufReader::new(source); let res = loop { - if nu_utils::ctrl_c::was_pressed(&ctrlc) { + if let Err(err) = signals.check(span) { bar.abandoned_msg("# Cancelled #".to_owned()); - return Ok(()); + return Err(err); } match reader.fill_buf() { @@ -555,7 +512,7 @@ fn stream_to_file( Ok(()) } } else { - copy_with_interrupt(&mut source, &mut file, span, ctrlc.as_deref())?; + copy_with_signals(source, file, span, signals)?; Ok(()) } } diff --git a/crates/nu-command/src/filesystem/touch.rs b/crates/nu-command/src/filesystem/touch.rs index 08843b11f0..2a1e128b04 100644 --- a/crates/nu-command/src/filesystem/touch.rs +++ b/crates/nu-command/src/filesystem/touch.rs @@ -1,10 +1,9 @@ use filetime::FileTime; -#[allow(deprecated)] -use nu_engine::{command_prelude::*, current_dir}; +use nu_engine::command_prelude::*; use nu_path::expand_path_with; use nu_protocol::NuGlob; -use std::{fs::OpenOptions, path::Path, time::SystemTime}; +use std::{fs::OpenOptions, time::SystemTime}; use super::util::get_rest_for_glob_pattern; @@ -36,12 +35,12 @@ impl Command for Touch { ) .switch( "modified", - "change the modification time of the file or directory. If no timestamp, date or reference file/directory is given, the current time is used", + "change the modification time of the file or directory. If no reference file/directory is given, the current time is used", Some('m'), ) .switch( "access", - "change the access time of the file or directory. If no timestamp, date or reference file/directory is given, the current time is used", + "change the access time of the file or directory. If no reference file/directory is given, the current time is used", Some('a'), ) .switch( @@ -69,6 +68,8 @@ impl Command for Touch { let no_create: bool = call.has_flag(engine_state, stack, "no-create")?; let files: Vec> = get_rest_for_glob_pattern(engine_state, stack, call, 0)?; + let cwd = engine_state.cwd(Some(stack))?; + if files.is_empty() { return Err(ShellError::MissingParameter { param_name: "requires file paths".to_string(), @@ -86,7 +87,7 @@ impl Command for Touch { } if let Some(reference) = reference { - let reference_path = Path::new(&reference.item); + let reference_path = nu_path::expand_path_with(reference.item, &cwd, true); if !reference_path.exists() { return Err(ShellError::FileNotFoundCustom { msg: "Reference path not found".into(), @@ -114,9 +115,6 @@ impl Command for Touch { })?; } - #[allow(deprecated)] - let cwd = current_dir(engine_state, stack)?; - for glob in files { let path = expand_path_with(glob.item.as_ref(), &cwd, glob.item.is_expand()); @@ -191,11 +189,6 @@ impl Command for Touch { example: r#"touch -m -r fixture.json d e"#, result: None, }, - Example { - description: r#"Changes the last accessed time of "fixture.json" to a date"#, - example: r#"touch -a -d "August 24, 2019; 12:30:30" fixture.json"#, - result: None, - }, ] } } diff --git a/crates/nu-command/src/filesystem/util.rs b/crates/nu-command/src/filesystem/util.rs index 1b755875bd..de32d204a0 100644 --- a/crates/nu-command/src/filesystem/util.rs +++ b/crates/nu-command/src/filesystem/util.rs @@ -1,6 +1,6 @@ use dialoguer::Input; use nu_engine::{command_prelude::*, get_eval_expression}; -use nu_protocol::{ast::Expr, FromValue, NuGlob}; +use nu_protocol::{FromValue, NuGlob}; use std::{ error::Error, path::{Path, PathBuf}, @@ -92,42 +92,19 @@ pub fn is_older(src: &Path, dst: &Path) -> Option { /// Get rest arguments from given `call`, starts with `starting_pos`. /// -/// It's similar to `call.rest`, except that it always returns NuGlob. And if input argument has -/// Type::Glob, the NuGlob is unquoted, which means it's required to expand. +/// It's similar to `call.rest`, except that it always returns NuGlob. pub fn get_rest_for_glob_pattern( engine_state: &EngineState, stack: &mut Stack, call: &Call, starting_pos: usize, ) -> Result>, ShellError> { - let mut output = vec![]; let eval_expression = get_eval_expression(engine_state); - for result in call.rest_iter_flattened(starting_pos, |expr| { - let result = eval_expression(engine_state, stack, expr); - match result { - Err(e) => Err(e), - Ok(result) => { - let span = result.span(); - // convert from string to quoted string if expr is a variable - // or string interpolation - match result { - Value::String { val, .. } - if matches!( - &expr.expr, - Expr::FullCellPath(_) | Expr::StringInterpolation(_) - ) => - { - // should not expand if given input type is not glob. - Ok(Value::glob(val, expr.ty != Type::Glob, span)) - } - other => Ok(other), - } - } - } - })? { - output.push(FromValue::from_value(result)?); - } - - Ok(output) + call.rest_iter_flattened(engine_state, stack, eval_expression, starting_pos)? + .into_iter() + // This used to be much more complex, but I think `FromValue` should be able to handle the + // nuance here. + .map(FromValue::from_value) + .collect() } diff --git a/crates/nu-command/src/filesystem/watch.rs b/crates/nu-command/src/filesystem/watch.rs index fda542c8a8..c9817de250 100644 --- a/crates/nu-command/src/filesystem/watch.rs +++ b/crates/nu-command/src/filesystem/watch.rs @@ -143,7 +143,6 @@ impl Command for Watch { None => RecursiveMode::Recursive, }; - let ctrlc_ref = &engine_state.ctrlc.clone(); let (tx, rx) = channel(); let mut debouncer = match new_debouncer(debounce_duration, None, tx) { @@ -256,7 +255,7 @@ impl Command for Watch { } Err(RecvTimeoutError::Timeout) => {} } - if nu_utils::ctrl_c::was_pressed(ctrlc_ref) { + if engine_state.signals().interrupted() { break; } } diff --git a/crates/nu-command/src/filters/append.rs b/crates/nu-command/src/filters/append.rs index af5bd49283..2064bdf1e8 100644 --- a/crates/nu-command/src/filters/append.rs +++ b/crates/nu-command/src/filters/append.rs @@ -116,7 +116,7 @@ only unwrap the outer list, and leave the variable's contents untouched."# Ok(input .into_iter() .chain(other.into_pipeline_data()) - .into_pipeline_data_with_metadata(call.head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata(call.head, engine_state.signals().clone(), metadata)) } } diff --git a/crates/nu-command/src/filters/compact.rs b/crates/nu-command/src/filters/compact.rs index 9a4e968e9b..f43d738ed1 100644 --- a/crates/nu-command/src/filters/compact.rs +++ b/crates/nu-command/src/filters/compact.rs @@ -140,7 +140,7 @@ pub fn compact( _ => true, } }, - engine_state.ctrlc.clone(), + engine_state.signals(), ) .map(|m| m.set_metadata(metadata)) } diff --git a/crates/nu-command/src/filters/default.rs b/crates/nu-command/src/filters/default.rs index 3eaa8d342e..a12d78f0b7 100644 --- a/crates/nu-command/src/filters/default.rs +++ b/crates/nu-command/src/filters/default.rs @@ -80,8 +80,6 @@ fn default( let value: Value = call.req(engine_state, stack, 0)?; let column: Option> = call.opt(engine_state, stack, 1)?; - let ctrlc = engine_state.ctrlc.clone(); - if let Some(column) = column { input .map( @@ -109,7 +107,7 @@ fn default( } _ => item, }, - ctrlc, + engine_state.signals(), ) .map(|x| x.set_metadata(metadata)) } else if input.is_nothing() { @@ -121,7 +119,7 @@ fn default( Value::Nothing { .. } => value.clone(), x => x, }, - ctrlc, + engine_state.signals(), ) .map(|x| x.set_metadata(metadata)) } diff --git a/crates/nu-command/src/filters/drop/column.rs b/crates/nu-command/src/filters/drop/column.rs index 01c13deee4..f168e6c2ca 100644 --- a/crates/nu-command/src/filters/drop/column.rs +++ b/crates/nu-command/src/filters/drop/column.rs @@ -102,7 +102,11 @@ fn drop_cols( Err(e) => Value::error(e, head), } })) - .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata( + head, + engine_state.signals().clone(), + metadata, + )) } else { Ok(PipelineData::Empty) } @@ -135,7 +139,7 @@ fn drop_cols( PipelineData::Empty => Ok(PipelineData::Empty), PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "table or record".into(), - wrong_type: "byte stream".into(), + wrong_type: stream.type_().describe().into(), dst_span: head, src_span: stream.span(), }), diff --git a/crates/nu-command/src/filters/drop/nth.rs b/crates/nu-command/src/filters/drop/nth.rs index a530de5aa1..a76c4f0d92 100644 --- a/crates/nu-command/src/filters/drop/nth.rs +++ b/crates/nu-command/src/filters/drop/nth.rs @@ -156,7 +156,7 @@ impl Command for DropNth { .take(start) .into_pipeline_data_with_metadata( head, - engine_state.ctrlc.clone(), + engine_state.signals().clone(), metadata, )) } @@ -177,7 +177,7 @@ impl Command for DropNth { rows, current: 0, } - .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata(head, engine_state.signals().clone(), metadata)) } } diff --git a/crates/nu-command/src/filters/each.rs b/crates/nu-command/src/filters/each.rs index a074f63abb..fa1bf390b8 100644 --- a/crates/nu-command/src/filters/each.rs +++ b/crates/nu-command/src/filters/each.rs @@ -140,7 +140,7 @@ with 'transpose' first."# } } }) - .into_pipeline_data(head, engine_state.ctrlc.clone())) + .into_pipeline_data(head, engine_state.signals().clone())) } PipelineData::ByteStream(stream, ..) => { if let Some(chunks) = stream.chunks() { @@ -171,7 +171,7 @@ with 'transpose' first."# } } }) - .into_pipeline_data(head, engine_state.ctrlc.clone())) + .into_pipeline_data(head, engine_state.signals().clone())) } else { Ok(PipelineData::Empty) } @@ -185,7 +185,7 @@ with 'transpose' first."# .and_then(|x| { x.filter( move |x| if !keep_empty { !x.is_nothing() } else { true }, - engine_state.ctrlc.clone(), + engine_state.signals(), ) }) .map(|data| data.set_metadata(metadata)) diff --git a/crates/nu-command/src/filters/enumerate.rs b/crates/nu-command/src/filters/enumerate.rs index 1034780657..df0f4e2fca 100644 --- a/crates/nu-command/src/filters/enumerate.rs +++ b/crates/nu-command/src/filters/enumerate.rs @@ -52,7 +52,6 @@ impl Command for Enumerate { ) -> Result { let head = call.head; let metadata = input.metadata(); - let ctrlc = engine_state.ctrlc.clone(); Ok(input .into_iter() @@ -66,7 +65,7 @@ impl Command for Enumerate { head, ) }) - .into_pipeline_data_with_metadata(head, ctrlc, metadata)) + .into_pipeline_data_with_metadata(head, engine_state.signals().clone(), metadata)) } } diff --git a/crates/nu-command/src/filters/every.rs b/crates/nu-command/src/filters/every.rs index 1202b4f7c9..664be33bed 100644 --- a/crates/nu-command/src/filters/every.rs +++ b/crates/nu-command/src/filters/every.rs @@ -78,7 +78,7 @@ impl Command for Every { None } }) - .into_pipeline_data_with_metadata(call.head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata(call.head, engine_state.signals().clone(), metadata)) } } diff --git a/crates/nu-command/src/filters/filter.rs b/crates/nu-command/src/filters/filter.rs index 1ba1508839..f2efaa3af3 100644 --- a/crates/nu-command/src/filters/filter.rs +++ b/crates/nu-command/src/filters/filter.rs @@ -72,7 +72,7 @@ a variable. On the other hand, the "row condition" syntax is not supported."# } } }) - .into_pipeline_data(head, engine_state.ctrlc.clone())) + .into_pipeline_data(head, engine_state.signals().clone())) } PipelineData::ByteStream(stream, ..) => { if let Some(chunks) = stream.chunks() { @@ -97,7 +97,7 @@ a variable. On the other hand, the "row condition" syntax is not supported."# } } }) - .into_pipeline_data(head, engine_state.ctrlc.clone())) + .into_pipeline_data(head, engine_state.signals().clone())) } else { Ok(PipelineData::Empty) } @@ -117,7 +117,7 @@ a variable. On the other hand, the "row condition" syntax is not supported."# Some(Value::error(err, span)) } } - .into_pipeline_data(head, engine_state.ctrlc.clone())) + .into_pipeline_data(head, engine_state.signals().clone())) } } .map(|data| data.set_metadata(metadata)) diff --git a/crates/nu-command/src/filters/find.rs b/crates/nu-command/src/filters/find.rs index dfdef66969..7669190f7e 100644 --- a/crates/nu-command/src/filters/find.rs +++ b/crates/nu-command/src/filters/find.rs @@ -69,9 +69,9 @@ impl Command for Find { result: None, }, Example { - description: "Search and highlight text for a term in a string", - example: r#"'Cargo.toml' | find toml"#, - result: Some(Value::test_string("\u{1b}[37mCargo.\u{1b}[0m\u{1b}[41;37mtoml\u{1b}[0m\u{1b}[37m\u{1b}[0m".to_owned())), + description: "Search and highlight text for a term in a string. Note that regular search is case insensitive", + example: r#"'Cargo.toml' | find cargo"#, + result: Some(Value::test_string("\u{1b}[37m\u{1b}[0m\u{1b}[41;37mCargo\u{1b}[0m\u{1b}[37m.toml\u{1b}[0m".to_owned())), }, Example { description: "Search a number or a file size in a list of numbers", @@ -213,7 +213,6 @@ fn find_with_regex( input: PipelineData, ) -> Result { let span = call.head; - let ctrlc = engine_state.ctrlc.clone(); let config = engine_state.get_config().clone(); let insensitive = call.has_flag(engine_state, stack, "ignore-case")?; @@ -246,7 +245,7 @@ fn find_with_regex( Value::List { vals, .. } => values_match_find(vals, &re, &config, invert), _ => false, }, - ctrlc, + engine_state.signals(), ) } @@ -349,18 +348,16 @@ fn find_with_rest_and_highlight( input: PipelineData, ) -> Result { let span = call.head; - let ctrlc = engine_state.ctrlc.clone(); - let engine_state = engine_state.clone(); let config = engine_state.get_config().clone(); let filter_config = engine_state.get_config().clone(); - let invert = call.has_flag(&engine_state, stack, "invert")?; - let terms = call.rest::(&engine_state, stack, 0)?; + let invert = call.has_flag(engine_state, stack, "invert")?; + let terms = call.rest::(engine_state, stack, 0)?; let lower_terms = terms .iter() .map(|v| Value::string(v.to_expanded_string("", &config).to_lowercase(), span)) .collect::>(); - let style_computer = StyleComputer::from_config(&engine_state, stack); + let style_computer = StyleComputer::from_config(engine_state, stack); // Currently, search results all use the same style. // Also note that this sample string is passed into user-written code (the closure that may or may not be // defined for "string"). @@ -369,7 +366,7 @@ fn find_with_rest_and_highlight( style_computer.compute("search_result", &Value::string("search result", span)); let cols_to_search_in_map: Vec<_> = call - .get_flag(&engine_state, stack, "columns")? + .get_flag(engine_state, stack, "columns")? .unwrap_or_default(); let cols_to_search_in_filter = cols_to_search_in_map.clone(); @@ -401,7 +398,7 @@ fn find_with_rest_and_highlight( _ => x, } }, - ctrlc.clone(), + engine_state.signals(), )? .filter( move |value| { @@ -414,7 +411,7 @@ fn find_with_rest_and_highlight( invert, ) }, - ctrlc, + engine_state.signals(), ), PipelineData::ListStream(stream, metadata) => { let stream = stream.modify(|iter| { @@ -457,9 +454,10 @@ fn find_with_rest_and_highlight( let mut output: Vec = vec![]; for line in lines { - let line = line?.to_lowercase(); + let line = line?; + let lower_val = line.to_lowercase(); for term in &terms { - if line.contains(term) { + if lower_val.contains(term) { output.push(Value::string( highlight_search_string( &line, diff --git a/crates/nu-command/src/filters/first.rs b/crates/nu-command/src/filters/first.rs index e581c3e84d..8f2e8db1b9 100644 --- a/crates/nu-command/src/filters/first.rs +++ b/crates/nu-command/src/filters/first.rs @@ -1,4 +1,6 @@ use nu_engine::command_prelude::*; +use nu_protocol::Signals; +use std::io::Read; #[derive(Clone)] pub struct First; @@ -132,8 +134,7 @@ fn first_helper( } } Value::Range { val, .. } => { - let ctrlc = engine_state.ctrlc.clone(); - let mut iter = val.into_range_iter(span, ctrlc.clone()); + let mut iter = val.into_range_iter(span, Signals::empty()); if return_single_element { if let Some(v) = iter.next() { Ok(v.into_pipeline_data()) @@ -141,9 +142,11 @@ fn first_helper( Err(ShellError::AccessEmptyContent { span: head }) } } else { - Ok(iter - .take(rows) - .into_pipeline_data_with_metadata(span, ctrlc, metadata)) + Ok(iter.take(rows).into_pipeline_data_with_metadata( + span, + engine_state.signals().clone(), + metadata, + )) } } // Propagate errors by explicitly matching them before the final case. @@ -170,12 +173,42 @@ fn first_helper( )) } } - PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary or range".into(), - wrong_type: "byte stream".into(), - dst_span: head, - src_span: stream.span(), - }), + PipelineData::ByteStream(stream, metadata) => { + if stream.type_().is_binary_coercible() { + let span = stream.span(); + if let Some(mut reader) = stream.reader() { + if return_single_element { + // Take a single byte + let mut byte = [0u8]; + if reader.read(&mut byte).err_span(span)? > 0 { + Ok(Value::int(byte[0] as i64, head).into_pipeline_data()) + } else { + Err(ShellError::AccessEmptyContent { span: head }) + } + } else { + // Just take 'rows' bytes off the stream, mimicking the binary behavior + Ok(PipelineData::ByteStream( + ByteStream::read( + reader.take(rows as u64), + head, + Signals::empty(), + ByteStreamType::Binary, + ), + metadata, + )) + } + } else { + Ok(PipelineData::Empty) + } + } else { + Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary or range".into(), + wrong_type: stream.type_().describe().into(), + dst_span: head, + src_span: stream.span(), + }) + } + } PipelineData::Empty => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), wrong_type: "null".into(), diff --git a/crates/nu-command/src/filters/flatten.rs b/crates/nu-command/src/filters/flatten.rs index ec86677af4..b4faec9589 100644 --- a/crates/nu-command/src/filters/flatten.rs +++ b/crates/nu-command/src/filters/flatten.rs @@ -127,7 +127,7 @@ fn flatten( input .flat_map( move |item| flat_value(&columns, item, flatten_all), - engine_state.ctrlc.clone(), + engine_state.signals(), ) .map(|x| x.set_metadata(metadata)) } diff --git a/crates/nu-command/src/filters/get.rs b/crates/nu-command/src/filters/get.rs index 07f0ea9440..9f7d76277d 100644 --- a/crates/nu-command/src/filters/get.rs +++ b/crates/nu-command/src/filters/get.rs @@ -62,7 +62,6 @@ If multiple cell paths are given, this will produce a list of values."# let mut rest: Vec = call.rest(engine_state, stack, 1)?; let ignore_errors = call.has_flag(engine_state, stack, "ignore-errors")?; let sensitive = call.has_flag(engine_state, stack, "sensitive")?; - let ctrlc = engine_state.ctrlc.clone(); let metadata = input.metadata(); if ignore_errors { @@ -89,7 +88,9 @@ If multiple cell paths are given, this will produce a list of values."# output.push(val?); } - Ok(output.into_iter().into_pipeline_data(span, ctrlc)) + Ok(output + .into_iter() + .into_pipeline_data(span, engine_state.signals().clone())) } .map(|x| x.set_metadata(metadata)) } diff --git a/crates/nu-command/src/filters/group.rs b/crates/nu-command/src/filters/group.rs index 821f35f34e..13b53850d2 100644 --- a/crates/nu-command/src/filters/group.rs +++ b/crates/nu-command/src/filters/group.rs @@ -55,18 +55,19 @@ impl Command for Group { ) -> Result { let head = call.head; let group_size: Spanned = call.req(engine_state, stack, 0)?; - let ctrlc = engine_state.ctrlc.clone(); let metadata = input.metadata(); - //FIXME: add in support for external redirection when engine-q supports it generally - let each_group_iterator = EachGroupIterator { group_size: group_size.item, input: Box::new(input.into_iter()), span: head, }; - Ok(each_group_iterator.into_pipeline_data_with_metadata(head, ctrlc, metadata)) + Ok(each_group_iterator.into_pipeline_data_with_metadata( + head, + engine_state.signals().clone(), + metadata, + )) } } diff --git a/crates/nu-command/src/filters/insert.rs b/crates/nu-command/src/filters/insert.rs index e8794304c8..3b47c4e6a9 100644 --- a/crates/nu-command/src/filters/insert.rs +++ b/crates/nu-command/src/filters/insert.rs @@ -222,7 +222,11 @@ fn insert( Ok(pre_elems .into_iter() .chain(stream) - .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata( + head, + engine_state.signals().clone(), + metadata, + )) } else if let Value::Closure { val, .. } = replacement { let mut closure = ClosureEval::new(engine_state, stack, *val); let stream = stream.map(move |mut value| { @@ -261,8 +265,8 @@ fn insert( type_name: "empty pipeline".to_string(), span: head, }), - PipelineData::ByteStream(..) => Err(ShellError::IncompatiblePathAccess { - type_name: "byte stream".to_string(), + PipelineData::ByteStream(stream, ..) => Err(ShellError::IncompatiblePathAccess { + type_name: stream.type_().describe().into(), span: head, }), } diff --git a/crates/nu-command/src/filters/interleave.rs b/crates/nu-command/src/filters/interleave.rs index 85a92741a1..9890fede1e 100644 --- a/crates/nu-command/src/filters/interleave.rs +++ b/crates/nu-command/src/filters/interleave.rs @@ -147,7 +147,7 @@ interleave // Now that threads are writing to the channel, we just return it as a stream Ok(rx .into_iter() - .into_pipeline_data(head, engine_state.ctrlc.clone())) + .into_pipeline_data(head, engine_state.signals().clone())) } } diff --git a/crates/nu-command/src/filters/items.rs b/crates/nu-command/src/filters/items.rs index 6afc0bc536..04a4c6d672 100644 --- a/crates/nu-command/src/filters/items.rs +++ b/crates/nu-command/src/filters/items.rs @@ -67,7 +67,7 @@ impl Command for Items { } } }) - .into_pipeline_data(head, engine_state.ctrlc.clone())) + .into_pipeline_data(head, engine_state.signals().clone())) } Value::Error { error, .. } => Err(*error), other => Err(ShellError::OnlySupportsThisInputType { @@ -86,7 +86,7 @@ impl Command for Items { }), PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "record".into(), - wrong_type: "byte stream".into(), + wrong_type: stream.type_().describe().into(), dst_span: call.head, src_span: stream.span(), }), diff --git a/crates/nu-command/src/filters/last.rs b/crates/nu-command/src/filters/last.rs index 7530126c26..bb5a75fecd 100644 --- a/crates/nu-command/src/filters/last.rs +++ b/crates/nu-command/src/filters/last.rs @@ -1,6 +1,5 @@ use nu_engine::command_prelude::*; - -use std::collections::VecDeque; +use std::{collections::VecDeque, io::Read}; #[derive(Clone)] pub struct Last; @@ -100,14 +99,10 @@ impl Command for Last { let mut buf = VecDeque::new(); for row in iterator { - if nu_utils::ctrl_c::was_pressed(&engine_state.ctrlc) { - return Err(ShellError::InterruptedByUser { span: Some(head) }); - } - + engine_state.signals().check(head)?; if buf.len() == rows { buf.pop_front(); } - buf.push_back(row); } @@ -160,12 +155,47 @@ impl Command for Last { }), } } - PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary or range".into(), - wrong_type: "byte stream".into(), - dst_span: head, - src_span: stream.span(), - }), + PipelineData::ByteStream(stream, ..) => { + if stream.type_().is_binary_coercible() { + let span = stream.span(); + if let Some(mut reader) = stream.reader() { + // Have to be a bit tricky here, but just consume into a VecDeque that we + // shrink to fit each time + const TAKE: u64 = 8192; + let mut buf = VecDeque::with_capacity(rows + TAKE as usize); + loop { + let taken = std::io::copy(&mut (&mut reader).take(TAKE), &mut buf) + .err_span(span)?; + if buf.len() > rows { + buf.drain(..(buf.len() - rows)); + } + if taken < TAKE { + // This must be EOF. + if return_single_element { + if !buf.is_empty() { + return Ok( + Value::int(buf[0] as i64, head).into_pipeline_data() + ); + } else { + return Err(ShellError::AccessEmptyContent { span: head }); + } + } else { + return Ok(Value::binary(buf, head).into_pipeline_data()); + } + } + } + } else { + Ok(PipelineData::Empty) + } + } else { + Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary or range".into(), + wrong_type: stream.type_().describe().into(), + dst_span: head, + src_span: stream.span(), + }) + } + } PipelineData::Empty => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), wrong_type: "null".into(), diff --git a/crates/nu-command/src/filters/lines.rs b/crates/nu-command/src/filters/lines.rs index 0b037dcaac..fc957e1c0e 100644 --- a/crates/nu-command/src/filters/lines.rs +++ b/crates/nu-command/src/filters/lines.rs @@ -26,7 +26,6 @@ impl Command for Lines { input: PipelineData, ) -> Result { let head = call.head; - let ctrlc = engine_state.ctrlc.clone(); let skip_empty = call.has_flag(engine_state, stack, "skip-empty")?; let span = input.span().unwrap_or(call.head); @@ -91,7 +90,7 @@ impl Command for Lines { Ok(line) => Value::string(line, head), Err(err) => Value::error(err, head), }) - .into_pipeline_data(head, ctrlc)) + .into_pipeline_data(head, engine_state.signals().clone())) } else { Ok(PipelineData::empty()) } diff --git a/crates/nu-command/src/filters/merge.rs b/crates/nu-command/src/filters/merge.rs index 8a3e613c56..191ed1706c 100644 --- a/crates/nu-command/src/filters/merge.rs +++ b/crates/nu-command/src/filters/merge.rs @@ -88,7 +88,6 @@ repeating this process with row 1, and so on."# let head = call.head; let merge_value: Value = call.req(engine_state, stack, 0)?; let metadata = input.metadata(); - let ctrlc = engine_state.ctrlc.clone(); match (&input, merge_value) { // table (list of records) @@ -110,7 +109,11 @@ repeating this process with row 1, and so on."# (Err(error), _) => Value::error(error, head), }); - Ok(res.into_pipeline_data_with_metadata(head, ctrlc, metadata)) + Ok(res.into_pipeline_data_with_metadata( + head, + engine_state.signals().clone(), + metadata, + )) } // record ( diff --git a/crates/nu-command/src/filters/move_.rs b/crates/nu-command/src/filters/move_.rs index d93368f291..6e826af050 100644 --- a/crates/nu-command/src/filters/move_.rs +++ b/crates/nu-command/src/filters/move_.rs @@ -144,7 +144,6 @@ impl Command for Move { }; let metadata = input.metadata(); - let ctrlc = engine_state.ctrlc.clone(); match input { PipelineData::Value(Value::List { .. }, ..) | PipelineData::ListStream { .. } => { @@ -158,7 +157,11 @@ impl Command for Move { Err(error) => Value::error(error, head), }); - Ok(res.into_pipeline_data_with_metadata(head, ctrlc, metadata)) + Ok(res.into_pipeline_data_with_metadata( + head, + engine_state.signals().clone(), + metadata, + )) } PipelineData::Value(Value::Record { val, .. }, ..) => { Ok(move_record_columns(&val, &columns, &before_or_after, head)? diff --git a/crates/nu-command/src/filters/par_each.rs b/crates/nu-command/src/filters/par_each.rs index af72895df6..958a7fbc76 100644 --- a/crates/nu-command/src/filters/par_each.rs +++ b/crates/nu-command/src/filters/par_each.rs @@ -1,6 +1,6 @@ use super::utils::chain_error_with_input; use nu_engine::{command_prelude::*, ClosureEvalOnce}; -use nu_protocol::engine::Closure; +use nu_protocol::{engine::Closure, Signals}; use rayon::prelude::*; #[derive(Clone)] @@ -158,12 +158,11 @@ impl Command for ParEach { }) .collect::>(); - apply_order(vec).into_pipeline_data(span, engine_state.ctrlc.clone()) + apply_order(vec).into_pipeline_data(span, engine_state.signals().clone()) })), Value::Range { val, .. } => Ok(create_pool(max_threads)?.install(|| { - let ctrlc = engine_state.ctrlc.clone(); let vec = val - .into_range_iter(span, ctrlc.clone()) + .into_range_iter(span, Signals::empty()) .enumerate() .par_bridge() .map(move |(index, value)| { @@ -184,7 +183,7 @@ impl Command for ParEach { }) .collect::>(); - apply_order(vec).into_pipeline_data(span, ctrlc) + apply_order(vec).into_pipeline_data(span, engine_state.signals().clone()) })), // This match allows non-iterables to be accepted, // which is currently considered undesirable (Nov 2022). @@ -212,7 +211,7 @@ impl Command for ParEach { }) .collect::>(); - apply_order(vec).into_pipeline_data(head, engine_state.ctrlc.clone()) + apply_order(vec).into_pipeline_data(head, engine_state.signals().clone()) })), PipelineData::ByteStream(stream, ..) => { if let Some(chunks) = stream.chunks() { @@ -236,14 +235,14 @@ impl Command for ParEach { }) .collect::>(); - apply_order(vec).into_pipeline_data(head, engine_state.ctrlc.clone()) + apply_order(vec).into_pipeline_data(head, engine_state.signals().clone()) })) } else { Ok(PipelineData::empty()) } } } - .and_then(|x| x.filter(|v| !v.is_nothing(), engine_state.ctrlc.clone())) + .and_then(|x| x.filter(|v| !v.is_nothing(), engine_state.signals())) .map(|data| data.set_metadata(metadata)) } } diff --git a/crates/nu-command/src/filters/prepend.rs b/crates/nu-command/src/filters/prepend.rs index f017420595..eeae0ef656 100644 --- a/crates/nu-command/src/filters/prepend.rs +++ b/crates/nu-command/src/filters/prepend.rs @@ -117,7 +117,7 @@ only unwrap the outer list, and leave the variable's contents untouched."# .into_pipeline_data() .into_iter() .chain(input) - .into_pipeline_data_with_metadata(call.head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata(call.head, engine_state.signals().clone(), metadata)) } } diff --git a/crates/nu-command/src/filters/range.rs b/crates/nu-command/src/filters/range.rs index c89a0b11f2..0d4e703bb1 100644 --- a/crates/nu-command/src/filters/range.rs +++ b/crates/nu-command/src/filters/range.rs @@ -106,7 +106,7 @@ impl Command for Range { Ok(PipelineData::Value(Value::nothing(head), None)) } else { let iter = v.into_iter().skip(from).take(to - from + 1); - Ok(iter.into_pipeline_data(head, engine_state.ctrlc.clone())) + Ok(iter.into_pipeline_data(head, engine_state.signals().clone())) } } else { let from = start as usize; @@ -116,7 +116,7 @@ impl Command for Range { Ok(PipelineData::Value(Value::nothing(head), None)) } else { let iter = input.into_iter().skip(from).take(to - from + 1); - Ok(iter.into_pipeline_data(head, engine_state.ctrlc.clone())) + Ok(iter.into_pipeline_data(head, engine_state.signals().clone())) } } .map(|x| x.set_metadata(metadata)) diff --git a/crates/nu-command/src/filters/reduce.rs b/crates/nu-command/src/filters/reduce.rs index fc808ca9af..87fbe3b23e 100644 --- a/crates/nu-command/src/filters/reduce.rs +++ b/crates/nu-command/src/filters/reduce.rs @@ -107,10 +107,7 @@ impl Command for Reduce { let mut closure = ClosureEval::new(engine_state, stack, closure); for value in iter { - if nu_utils::ctrl_c::was_pressed(&engine_state.ctrlc) { - break; - } - + engine_state.signals().check(head)?; acc = closure .add_arg(value) .add_arg(acc) diff --git a/crates/nu-command/src/filters/rename.rs b/crates/nu-command/src/filters/rename.rs index b803bd8567..1e7ce34028 100644 --- a/crates/nu-command/src/filters/rename.rs +++ b/crates/nu-command/src/filters/rename.rs @@ -221,7 +221,7 @@ fn rename( ), } }, - engine_state.ctrlc.clone(), + engine_state.signals(), ) .map(|data| data.set_metadata(metadata)) } diff --git a/crates/nu-command/src/filters/reverse.rs b/crates/nu-command/src/filters/reverse.rs index f40e5f3be4..18891637ab 100644 --- a/crates/nu-command/src/filters/reverse.rs +++ b/crates/nu-command/src/filters/reverse.rs @@ -63,7 +63,7 @@ impl Command for Reverse { let metadata = input.metadata(); let values = input.into_iter_strict(head)?.collect::>(); let iter = values.into_iter().rev(); - Ok(iter.into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) + Ok(iter.into_pipeline_data_with_metadata(head, engine_state.signals().clone(), metadata)) } } diff --git a/crates/nu-command/src/filters/select.rs b/crates/nu-command/src/filters/select.rs index 3514ac6be7..2ca04b3999 100644 --- a/crates/nu-command/src/filters/select.rs +++ b/crates/nu-command/src/filters/select.rs @@ -215,7 +215,11 @@ fn select( rows: unique_rows.into_iter().peekable(), current: 0, } - .into_pipeline_data_with_metadata(call_span, engine_state.ctrlc.clone(), metadata) + .into_pipeline_data_with_metadata( + call_span, + engine_state.signals().clone(), + metadata, + ) } else { input }; @@ -255,7 +259,7 @@ fn select( Ok(output.into_iter().into_pipeline_data_with_metadata( call_span, - engine_state.ctrlc.clone(), + engine_state.signals().clone(), metadata, )) } @@ -304,7 +308,7 @@ fn select( Ok(values.into_pipeline_data_with_metadata( call_span, - engine_state.ctrlc.clone(), + engine_state.signals().clone(), metadata, )) } diff --git a/crates/nu-command/src/filters/shuffle.rs b/crates/nu-command/src/filters/shuffle.rs index 9e023b86c6..ec4bf8c454 100644 --- a/crates/nu-command/src/filters/shuffle.rs +++ b/crates/nu-command/src/filters/shuffle.rs @@ -33,7 +33,11 @@ impl Command for Shuffle { let mut values = input.into_iter_strict(call.head)?.collect::>(); values.shuffle(&mut thread_rng()); let iter = values.into_iter(); - Ok(iter.into_pipeline_data_with_metadata(call.head, engine_state.ctrlc.clone(), metadata)) + Ok(iter.into_pipeline_data_with_metadata( + call.head, + engine_state.signals().clone(), + metadata, + )) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/filters/skip/skip_.rs b/crates/nu-command/src/filters/skip/skip_.rs index 9048b34a58..b64f438858 100644 --- a/crates/nu-command/src/filters/skip/skip_.rs +++ b/crates/nu-command/src/filters/skip/skip_.rs @@ -1,4 +1,6 @@ use nu_engine::command_prelude::*; +use nu_protocol::Signals; +use std::io::{self, Read}; #[derive(Clone)] pub struct Skip; @@ -12,6 +14,7 @@ impl Command for Skip { Signature::build(self.name()) .input_output_types(vec![ (Type::table(), Type::table()), + (Type::Binary, Type::Binary), ( Type::List(Box::new(Type::Any)), Type::List(Box::new(Type::Any)), @@ -51,6 +54,11 @@ impl Command for Skip { "editions" => Value::test_int(2021), })])), }, + Example { + description: "Skip 2 bytes of a binary value", + example: "0x[01 23 45 67] | skip 2", + result: Some(Value::test_binary(vec![0x45, 0x67])), + }, ] } fn run( @@ -83,16 +91,36 @@ impl Command for Skip { } None => 1, }; - - let ctrlc = engine_state.ctrlc.clone(); let input_span = input.span().unwrap_or(call.head); match input { - PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary or range".into(), - wrong_type: "byte stream".into(), - dst_span: call.head, - src_span: stream.span(), - }), + PipelineData::ByteStream(stream, metadata) => { + if stream.type_().is_binary_coercible() { + let span = stream.span(); + if let Some(mut reader) = stream.reader() { + // Copy the number of skipped bytes into the sink before proceeding + io::copy(&mut (&mut reader).take(n as u64), &mut io::sink()) + .err_span(span)?; + Ok(PipelineData::ByteStream( + ByteStream::read( + reader, + call.head, + Signals::empty(), + ByteStreamType::Binary, + ), + metadata, + )) + } else { + Ok(PipelineData::Empty) + } + } else { + Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary or range".into(), + wrong_type: stream.type_().describe().into(), + dst_span: call.head, + src_span: stream.span(), + }) + } + } PipelineData::Value(Value::Binary { val, .. }, metadata) => { let bytes = val.into_iter().skip(n).collect::>(); Ok(Value::binary(bytes, input_span).into_pipeline_data_with_metadata(metadata)) @@ -100,7 +128,11 @@ impl Command for Skip { _ => Ok(input .into_iter_strict(call.head)? .skip(n) - .into_pipeline_data_with_metadata(input_span, ctrlc, metadata)), + .into_pipeline_data_with_metadata( + input_span, + engine_state.signals().clone(), + metadata, + )), } } } diff --git a/crates/nu-command/src/filters/skip/skip_until.rs b/crates/nu-command/src/filters/skip/skip_until.rs index bb36785e00..72cae739af 100644 --- a/crates/nu-command/src/filters/skip/skip_until.rs +++ b/crates/nu-command/src/filters/skip/skip_until.rs @@ -89,7 +89,7 @@ impl Command for SkipUntil { .map(|cond| cond.is_false()) .unwrap_or(false) }) - .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata(head, engine_state.signals().clone(), metadata)) } } diff --git a/crates/nu-command/src/filters/skip/skip_while.rs b/crates/nu-command/src/filters/skip/skip_while.rs index 2747ea6f97..ea9c12bf6a 100644 --- a/crates/nu-command/src/filters/skip/skip_while.rs +++ b/crates/nu-command/src/filters/skip/skip_while.rs @@ -94,7 +94,7 @@ impl Command for SkipWhile { .map(|cond| cond.is_true()) .unwrap_or(false) }) - .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata(head, engine_state.signals().clone(), metadata)) } } diff --git a/crates/nu-command/src/filters/sort.rs b/crates/nu-command/src/filters/sort.rs index 965b997355..179235302d 100644 --- a/crates/nu-command/src/filters/sort.rs +++ b/crates/nu-command/src/filters/sort.rs @@ -173,7 +173,7 @@ impl Command for Sort { let iter = vec.into_iter(); Ok(iter.into_pipeline_data_with_metadata( head, - engine_state.ctrlc.clone(), + engine_state.signals().clone(), metadata, )) } diff --git a/crates/nu-command/src/filters/sort_by.rs b/crates/nu-command/src/filters/sort_by.rs index f3f715bc9d..e832255a26 100644 --- a/crates/nu-command/src/filters/sort_by.rs +++ b/crates/nu-command/src/filters/sort_by.rs @@ -100,7 +100,7 @@ impl Command for SortBy { } let iter = vec.into_iter(); - Ok(iter.into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) + Ok(iter.into_pipeline_data_with_metadata(head, engine_state.signals().clone(), metadata)) } } diff --git a/crates/nu-command/src/filters/split_by.rs b/crates/nu-command/src/filters/split_by.rs index 0d3bf1cd30..419119fe0c 100644 --- a/crates/nu-command/src/filters/split_by.rs +++ b/crates/nu-command/src/filters/split_by.rs @@ -130,7 +130,7 @@ pub fn split( Some(group_key) => Ok(group_key.coerce_string()?), None => Err(ShellError::CantFindColumn { col_name: column_name.item.to_string(), - span: column_name.span, + span: Some(column_name.span), src_span: row.span(), }), } diff --git a/crates/nu-command/src/filters/take/take_.rs b/crates/nu-command/src/filters/take/take_.rs index 12840aa8d6..1876244bdd 100644 --- a/crates/nu-command/src/filters/take/take_.rs +++ b/crates/nu-command/src/filters/take/take_.rs @@ -1,4 +1,6 @@ use nu_engine::command_prelude::*; +use nu_protocol::Signals; +use std::io::Read; #[derive(Clone)] pub struct Take; @@ -45,7 +47,6 @@ impl Command for Take { let head = call.head; let rows_desired: usize = call.req(engine_state, stack, 0)?; - let ctrlc = engine_state.ctrlc.clone(); let metadata = input.metadata(); match input { @@ -55,15 +56,23 @@ impl Command for Take { Value::List { vals, .. } => Ok(vals .into_iter() .take(rows_desired) - .into_pipeline_data_with_metadata(head, ctrlc, metadata)), + .into_pipeline_data_with_metadata( + head, + engine_state.signals().clone(), + metadata, + )), Value::Binary { val, .. } => { let slice: Vec = val.into_iter().take(rows_desired).collect(); Ok(PipelineData::Value(Value::binary(slice, span), metadata)) } Value::Range { val, .. } => Ok(val - .into_range_iter(span, ctrlc.clone()) + .into_range_iter(span, Signals::empty()) .take(rows_desired) - .into_pipeline_data_with_metadata(head, ctrlc, metadata)), + .into_pipeline_data_with_metadata( + head, + engine_state.signals().clone(), + metadata, + )), // Propagate errors by explicitly matching them before the final case. Value::Error { error, .. } => Err(*error), other => Err(ShellError::OnlySupportsThisInputType { @@ -78,12 +87,31 @@ impl Command for Take { stream.modify(|iter| iter.take(rows_desired)), metadata, )), - PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "list, binary or range".into(), - wrong_type: "byte stream".into(), - dst_span: head, - src_span: stream.span(), - }), + PipelineData::ByteStream(stream, metadata) => { + if stream.type_().is_binary_coercible() { + if let Some(reader) = stream.reader() { + // Just take 'rows' bytes off the stream, mimicking the binary behavior + Ok(PipelineData::ByteStream( + ByteStream::read( + reader.take(rows_desired as u64), + head, + Signals::empty(), + ByteStreamType::Binary, + ), + metadata, + )) + } else { + Ok(PipelineData::Empty) + } + } else { + Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "list, binary or range".into(), + wrong_type: stream.type_().describe().into(), + dst_span: head, + src_span: stream.span(), + }) + } + } PipelineData::Empty => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "list, binary or range".into(), wrong_type: "null".into(), diff --git a/crates/nu-command/src/filters/take/take_until.rs b/crates/nu-command/src/filters/take/take_until.rs index 0df2407cb1..c7debf5dee 100644 --- a/crates/nu-command/src/filters/take/take_until.rs +++ b/crates/nu-command/src/filters/take/take_until.rs @@ -85,7 +85,7 @@ impl Command for TakeUntil { .map(|cond| cond.is_false()) .unwrap_or(false) }) - .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata(head, engine_state.signals().clone(), metadata)) } } diff --git a/crates/nu-command/src/filters/take/take_while.rs b/crates/nu-command/src/filters/take/take_while.rs index 7c282ac38a..b8045080ea 100644 --- a/crates/nu-command/src/filters/take/take_while.rs +++ b/crates/nu-command/src/filters/take/take_while.rs @@ -85,7 +85,7 @@ impl Command for TakeWhile { .map(|cond| cond.is_true()) .unwrap_or(false) }) - .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata(head, engine_state.signals().clone(), metadata)) } } diff --git a/crates/nu-command/src/filters/tee.rs b/crates/nu-command/src/filters/tee.rs index 936dee5c79..663b2f19af 100644 --- a/crates/nu-command/src/filters/tee.rs +++ b/crates/nu-command/src/filters/tee.rs @@ -1,15 +1,11 @@ use nu_engine::{command_prelude::*, get_eval_block_with_early_return}; use nu_protocol::{ - byte_stream::copy_with_interrupt, engine::Closure, process::ChildPipe, ByteStream, - ByteStreamSource, OutDest, + byte_stream::copy_with_signals, engine::Closure, process::ChildPipe, ByteStream, + ByteStreamSource, OutDest, PipelineMetadata, Signals, }; use std::{ io::{self, Read, Write}, - sync::{ - atomic::AtomicBool, - mpsc::{self, Sender}, - Arc, - }, + sync::mpsc::{self, Sender}, thread::{self, JoinHandle}, }; @@ -103,10 +99,13 @@ use it in your pipeline."# if let PipelineData::ByteStream(stream, metadata) = input { let span = stream.span(); - let ctrlc = engine_state.ctrlc.clone(); - let eval_block = { - let metadata = metadata.clone(); - move |stream| eval_block(PipelineData::ByteStream(stream, metadata)) + let type_ = stream.type_(); + + let info = StreamInfo { + span, + signals: engine_state.signals().clone(), + type_, + metadata: metadata.clone(), }; match stream.into_source() { @@ -115,10 +114,11 @@ use it in your pipeline."# return stderr_misuse(span, head); } - let tee = IoTee::new(read, span, eval_block)?; + let tee_thread = spawn_tee(info, eval_block)?; + let tee = IoTee::new(read, tee_thread); Ok(PipelineData::ByteStream( - ByteStream::read(tee, span, ctrlc), + ByteStream::read(tee, span, engine_state.signals().clone(), type_), metadata, )) } @@ -127,44 +127,32 @@ use it in your pipeline."# return stderr_misuse(span, head); } - let tee = IoTee::new(file, span, eval_block)?; + let tee_thread = spawn_tee(info, eval_block)?; + let tee = IoTee::new(file, tee_thread); Ok(PipelineData::ByteStream( - ByteStream::read(tee, span, ctrlc), + ByteStream::read(tee, span, engine_state.signals().clone(), type_), metadata, )) } ByteStreamSource::Child(mut child) => { let stderr_thread = if use_stderr { let stderr_thread = if let Some(stderr) = child.stderr.take() { + let tee_thread = spawn_tee(info.clone(), eval_block)?; + let tee = IoTee::new(stderr, tee_thread); match stack.stderr() { OutDest::Pipe | OutDest::Capture => { - let tee = IoTee::new(stderr, span, eval_block)?; child.stderr = Some(ChildPipe::Tee(Box::new(tee))); - None + Ok(None) } - OutDest::Null => Some(tee_pipe_on_thread( - stderr, - io::sink(), - span, - ctrlc.as_ref(), - eval_block, - )?), - OutDest::Inherit => Some(tee_pipe_on_thread( - stderr, - io::stderr(), - span, - ctrlc.as_ref(), - eval_block, - )?), - OutDest::File(file) => Some(tee_pipe_on_thread( - stderr, - file.clone(), - span, - ctrlc.as_ref(), - eval_block, - )?), - } + OutDest::Null => copy_on_thread(tee, io::sink(), &info).map(Some), + OutDest::Inherit => { + copy_on_thread(tee, io::stderr(), &info).map(Some) + } + OutDest::File(file) => { + copy_on_thread(tee, file.clone(), &info).map(Some) + } + }? } else { None }; @@ -175,37 +163,29 @@ use it in your pipeline."# child.stdout = Some(stdout); Ok(()) } - OutDest::Null => { - copy_pipe(stdout, io::sink(), span, ctrlc.as_deref()) - } - OutDest::Inherit => { - copy_pipe(stdout, io::stdout(), span, ctrlc.as_deref()) - } - OutDest::File(file) => { - copy_pipe(stdout, file.as_ref(), span, ctrlc.as_deref()) - } + OutDest::Null => copy_pipe(stdout, io::sink(), &info), + OutDest::Inherit => copy_pipe(stdout, io::stdout(), &info), + OutDest::File(file) => copy_pipe(stdout, file.as_ref(), &info), }?; } stderr_thread } else { let stderr_thread = if let Some(stderr) = child.stderr.take() { + let info = info.clone(); match stack.stderr() { OutDest::Pipe | OutDest::Capture => { child.stderr = Some(stderr); Ok(None) } OutDest::Null => { - copy_pipe_on_thread(stderr, io::sink(), span, ctrlc.as_ref()) - .map(Some) + copy_pipe_on_thread(stderr, io::sink(), &info).map(Some) } OutDest::Inherit => { - copy_pipe_on_thread(stderr, io::stderr(), span, ctrlc.as_ref()) - .map(Some) + copy_pipe_on_thread(stderr, io::stderr(), &info).map(Some) } OutDest::File(file) => { - copy_pipe_on_thread(stderr, file.clone(), span, ctrlc.as_ref()) - .map(Some) + copy_pipe_on_thread(stderr, file.clone(), &info).map(Some) } }? } else { @@ -213,29 +193,16 @@ use it in your pipeline."# }; if let Some(stdout) = child.stdout.take() { + let tee_thread = spawn_tee(info.clone(), eval_block)?; + let tee = IoTee::new(stdout, tee_thread); match stack.stdout() { OutDest::Pipe | OutDest::Capture => { - let tee = IoTee::new(stdout, span, eval_block)?; child.stdout = Some(ChildPipe::Tee(Box::new(tee))); Ok(()) } - OutDest::Null => { - tee_pipe(stdout, io::sink(), span, ctrlc.as_deref(), eval_block) - } - OutDest::Inherit => tee_pipe( - stdout, - io::stdout(), - span, - ctrlc.as_deref(), - eval_block, - ), - OutDest::File(file) => tee_pipe( - stdout, - file.as_ref(), - span, - ctrlc.as_deref(), - eval_block, - ), + OutDest::Null => copy(tee, io::sink(), &info), + OutDest::Inherit => copy(tee, io::stdout(), &info), + OutDest::File(file) => copy(tee, file.as_ref(), &info), }?; } @@ -262,19 +229,19 @@ use it in your pipeline."# } let span = input.span().unwrap_or(head); - let ctrlc = engine_state.ctrlc.clone(); let metadata = input.metadata(); let metadata_clone = metadata.clone(); + let signals = engine_state.signals().clone(); Ok(tee(input.into_iter(), move |rx| { - let input = rx.into_pipeline_data_with_metadata(span, ctrlc, metadata_clone); + let input = rx.into_pipeline_data_with_metadata(span, signals, metadata_clone); eval_block(input) }) .err_span(call.head)? .map(move |result| result.unwrap_or_else(|err| Value::error(err, closure_span))) .into_pipeline_data_with_metadata( span, - engine_state.ctrlc.clone(), + engine_state.signals().clone(), metadata, )) } @@ -350,7 +317,7 @@ where fn stderr_misuse(span: Span, head: Span) -> Result { Err(ShellError::UnsupportedInput { msg: "--stderr can only be used on external commands".into(), - input: "the input to `tee` is not an external commands".into(), + input: "the input to `tee` is not an external command".into(), msg_span: head, input_span: span, }) @@ -363,23 +330,12 @@ struct IoTee { } impl IoTee { - fn new( - reader: R, - span: Span, - eval_block: impl FnOnce(ByteStream) -> Result<(), ShellError> + Send + 'static, - ) -> Result { - let (sender, receiver) = mpsc::channel(); - - let thread = thread::Builder::new() - .name("tee".into()) - .spawn(move || eval_block(ByteStream::from_iter(receiver, span, None))) - .err_span(span)?; - - Ok(Self { + fn new(reader: R, tee: TeeThread) -> Self { + Self { reader, - sender: Some(sender), - thread: Some(thread), - }) + sender: Some(tee.sender), + thread: Some(tee.thread), + } } } @@ -411,68 +367,79 @@ impl Read for IoTee { } } -fn tee_pipe( - pipe: ChildPipe, - mut dest: impl Write, +struct TeeThread { + sender: Sender>, + thread: JoinHandle>, +} + +fn spawn_tee( + info: StreamInfo, + mut eval_block: impl FnMut(PipelineData) -> Result<(), ShellError> + Send + 'static, +) -> Result { + let (sender, receiver) = mpsc::channel(); + + let thread = thread::Builder::new() + .name("tee".into()) + .spawn(move || { + // We use Signals::empty() here because we assume there already is a Signals on the other side + let stream = ByteStream::from_iter( + receiver.into_iter(), + info.span, + Signals::empty(), + info.type_, + ); + eval_block(PipelineData::ByteStream(stream, info.metadata)) + }) + .err_span(info.span)?; + + Ok(TeeThread { sender, thread }) +} + +#[derive(Clone)] +struct StreamInfo { span: Span, - ctrlc: Option<&AtomicBool>, - eval_block: impl FnOnce(ByteStream) -> Result<(), ShellError> + Send + 'static, -) -> Result<(), ShellError> { - match pipe { - ChildPipe::Pipe(pipe) => { - let mut tee = IoTee::new(pipe, span, eval_block)?; - copy_with_interrupt(&mut tee, &mut dest, span, ctrlc)?; - } - ChildPipe::Tee(tee) => { - let mut tee = IoTee::new(tee, span, eval_block)?; - copy_with_interrupt(&mut tee, &mut dest, span, ctrlc)?; - } - } + signals: Signals, + type_: ByteStreamType, + metadata: Option, +} + +fn copy(src: impl Read, dest: impl Write, info: &StreamInfo) -> Result<(), ShellError> { + copy_with_signals(src, dest, info.span, &info.signals)?; Ok(()) } -fn tee_pipe_on_thread( - pipe: ChildPipe, +fn copy_pipe(pipe: ChildPipe, dest: impl Write, info: &StreamInfo) -> Result<(), ShellError> { + match pipe { + ChildPipe::Pipe(pipe) => copy(pipe, dest, info), + ChildPipe::Tee(tee) => copy(tee, dest, info), + } +} + +fn copy_on_thread( + src: impl Read + Send + 'static, dest: impl Write + Send + 'static, - span: Span, - ctrlc: Option<&Arc>, - eval_block: impl FnOnce(ByteStream) -> Result<(), ShellError> + Send + 'static, + info: &StreamInfo, ) -> Result>, ShellError> { - let ctrlc = ctrlc.cloned(); + let span = info.span; + let signals = info.signals.clone(); thread::Builder::new() - .name("stderr tee".into()) - .spawn(move || tee_pipe(pipe, dest, span, ctrlc.as_deref(), eval_block)) + .name("stderr copier".into()) + .spawn(move || { + copy_with_signals(src, dest, span, &signals)?; + Ok(()) + }) .map_err(|e| e.into_spanned(span).into()) } -fn copy_pipe( - pipe: ChildPipe, - mut dest: impl Write, - span: Span, - ctrlc: Option<&AtomicBool>, -) -> Result<(), ShellError> { - match pipe { - ChildPipe::Pipe(mut pipe) => { - copy_with_interrupt(&mut pipe, &mut dest, span, ctrlc)?; - } - ChildPipe::Tee(mut tee) => { - copy_with_interrupt(&mut tee, &mut dest, span, ctrlc)?; - } - } - Ok(()) -} - fn copy_pipe_on_thread( pipe: ChildPipe, dest: impl Write + Send + 'static, - span: Span, - ctrlc: Option<&Arc>, + info: &StreamInfo, ) -> Result>, ShellError> { - let ctrlc = ctrlc.cloned(); - thread::Builder::new() - .name("stderr copier".into()) - .spawn(move || copy_pipe(pipe, dest, span, ctrlc.as_deref())) - .map_err(|e| e.into_spanned(span).into()) + match pipe { + ChildPipe::Pipe(pipe) => copy_on_thread(pipe, dest, info), + ChildPipe::Tee(tee) => copy_on_thread(tee, dest, info), + } } #[test] diff --git a/crates/nu-command/src/filters/transpose.rs b/crates/nu-command/src/filters/transpose.rs index 49caa56d18..4a14c1aea2 100644 --- a/crates/nu-command/src/filters/transpose.rs +++ b/crates/nu-command/src/filters/transpose.rs @@ -149,31 +149,30 @@ pub fn transpose( if !args.rest.is_empty() && args.header_row { return Err(ShellError::IncompatibleParametersSingle { msg: "Can not provide header names and use `--header-row`".into(), - span: call.get_named_arg("header-row").expect("has flag").span, + span: call.get_flag_span(stack, "header-row").expect("has flag"), }); } if !args.header_row && args.keep_all { return Err(ShellError::IncompatibleParametersSingle { msg: "Can only be used with `--header-row`(`-r`)".into(), - span: call.get_named_arg("keep-all").expect("has flag").span, + span: call.get_flag_span(stack, "keep-all").expect("has flag"), }); } if !args.header_row && args.keep_last { return Err(ShellError::IncompatibleParametersSingle { msg: "Can only be used with `--header-row`(`-r`)".into(), - span: call.get_named_arg("keep-last").expect("has flag").span, + span: call.get_flag_span(stack, "keep-last").expect("has flag"), }); } if args.keep_all && args.keep_last { return Err(ShellError::IncompatibleParameters { left_message: "can't use `--keep-last` at the same time".into(), - left_span: call.get_named_arg("keep-last").expect("has flag").span, + left_span: call.get_flag_span(stack, "keep-last").expect("has flag"), right_message: "because of `--keep-all`".into(), - right_span: call.get_named_arg("keep-all").expect("has flag").span, + right_span: call.get_flag_span(stack, "keep-all").expect("has flag"), }); } - let ctrlc = engine_state.ctrlc.clone(); let metadata = input.metadata(); let input: Vec<_> = input.into_iter().collect(); @@ -284,7 +283,11 @@ pub fn transpose( metadata, )) } else { - Ok(result_data.into_pipeline_data_with_metadata(name, ctrlc, metadata)) + Ok(result_data.into_pipeline_data_with_metadata( + name, + engine_state.signals().clone(), + metadata, + )) } } diff --git a/crates/nu-command/src/filters/uniq.rs b/crates/nu-command/src/filters/uniq.rs index 7d71d868f0..99abb4e152 100644 --- a/crates/nu-command/src/filters/uniq.rs +++ b/crates/nu-command/src/filters/uniq.rs @@ -241,18 +241,18 @@ pub fn uniq( item_mapper: Box ValueCounter>, metadata: Option, ) -> Result { - let ctrlc = engine_state.ctrlc.clone(); let head = call.head; let flag_show_count = call.has_flag(engine_state, stack, "count")?; let flag_show_repeated = call.has_flag(engine_state, stack, "repeated")?; let flag_ignore_case = call.has_flag(engine_state, stack, "ignore-case")?; let flag_only_uniques = call.has_flag(engine_state, stack, "unique")?; + let signals = engine_state.signals().clone(); let uniq_values = input .into_iter() .enumerate() .map_while(|(index, item)| { - if nu_utils::ctrl_c::was_pressed(&ctrlc) { + if signals.interrupted() { return None; } Some(item_mapper(ItemMapperState { diff --git a/crates/nu-command/src/filters/uniq_by.rs b/crates/nu-command/src/filters/uniq_by.rs index 7bbeb0afe2..e1a502b06b 100644 --- a/crates/nu-command/src/filters/uniq_by.rs +++ b/crates/nu-command/src/filters/uniq_by.rs @@ -123,7 +123,7 @@ fn validate(vec: &[Value], columns: &[String], span: Span) -> Result<(), ShellEr if let Some(nonexistent) = nonexistent_column(columns, record.columns()) { return Err(ShellError::CantFindColumn { col_name: nonexistent, - span, + span: Some(span), src_span: val_span, }); } diff --git a/crates/nu-command/src/filters/update.rs b/crates/nu-command/src/filters/update.rs index 0d914d2d8e..d5ef0825ec 100644 --- a/crates/nu-command/src/filters/update.rs +++ b/crates/nu-command/src/filters/update.rs @@ -187,7 +187,11 @@ fn update( Ok(pre_elems .into_iter() .chain(stream) - .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata( + head, + engine_state.signals().clone(), + metadata, + )) } else if let Value::Closure { val, .. } = replacement { let mut closure = ClosureEval::new(engine_state, stack, *val); let stream = stream.map(move |mut value| { @@ -225,8 +229,8 @@ fn update( type_name: "empty pipeline".to_string(), span: head, }), - PipelineData::ByteStream(..) => Err(ShellError::IncompatiblePathAccess { - type_name: "byte stream".to_string(), + PipelineData::ByteStream(stream, ..) => Err(ShellError::IncompatiblePathAccess { + type_name: stream.type_().describe().into(), span: head, }), } diff --git a/crates/nu-command/src/filters/upsert.rs b/crates/nu-command/src/filters/upsert.rs index 4313addd89..af2e6c74b1 100644 --- a/crates/nu-command/src/filters/upsert.rs +++ b/crates/nu-command/src/filters/upsert.rs @@ -247,7 +247,11 @@ fn upsert( Ok(pre_elems .into_iter() .chain(stream) - .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata( + head, + engine_state.signals().clone(), + metadata, + )) } else if let Value::Closure { val, .. } = replacement { let mut closure = ClosureEval::new(engine_state, stack, *val); let stream = stream.map(move |mut value| { @@ -285,8 +289,8 @@ fn upsert( type_name: "empty pipeline".to_string(), span: head, }), - PipelineData::ByteStream(..) => Err(ShellError::IncompatiblePathAccess { - type_name: "byte stream".to_string(), + PipelineData::ByteStream(stream, ..) => Err(ShellError::IncompatiblePathAccess { + type_name: stream.type_().describe().into(), span: head, }), } diff --git a/crates/nu-command/src/filters/utils.rs b/crates/nu-command/src/filters/utils.rs index 8d9b1300f6..4c67667e8e 100644 --- a/crates/nu-command/src/filters/utils.rs +++ b/crates/nu-command/src/filters/utils.rs @@ -1,7 +1,6 @@ use nu_engine::{CallExt, ClosureEval}; use nu_protocol::{ - ast::Call, - engine::{Closure, EngineState, Stack}, + engine::{Call, Closure, EngineState, Stack}, IntoPipelineData, PipelineData, ShellError, Span, Value, }; @@ -32,10 +31,7 @@ pub fn boolean_fold( let mut closure = ClosureEval::new(engine_state, stack, closure); for value in input { - if nu_utils::ctrl_c::was_pressed(&engine_state.ctrlc) { - break; - } - + engine_state.signals().check(head)?; let pred = closure.run_with_value(value)?.into_value(head)?.is_true(); if pred == accumulator { diff --git a/crates/nu-command/src/filters/values.rs b/crates/nu-command/src/filters/values.rs index ed33ebf643..9a08f1168c 100644 --- a/crates/nu-command/src/filters/values.rs +++ b/crates/nu-command/src/filters/values.rs @@ -134,7 +134,7 @@ fn values( head: Span, input: PipelineData, ) -> Result { - let ctrlc = engine_state.ctrlc.clone(); + let signals = engine_state.signals().clone(); let metadata = input.metadata(); match input { PipelineData::Empty => Ok(PipelineData::Empty), @@ -144,7 +144,7 @@ fn values( Value::List { vals, .. } => match get_values(&vals, head, span) { Ok(cols) => Ok(cols .into_iter() - .into_pipeline_data_with_metadata(head, ctrlc, metadata)), + .into_pipeline_data_with_metadata(head, signals, metadata)), Err(err) => Err(err), }, Value::Custom { val, .. } => { @@ -152,7 +152,7 @@ fn values( match get_values(&[input_as_base_value], head, span) { Ok(cols) => Ok(cols .into_iter() - .into_pipeline_data_with_metadata(head, ctrlc, metadata)), + .into_pipeline_data_with_metadata(head, signals, metadata)), Err(err) => Err(err), } } @@ -160,7 +160,7 @@ fn values( .values() .cloned() .collect::>() - .into_pipeline_data_with_metadata(head, ctrlc, metadata)), + .into_pipeline_data_with_metadata(head, signals, metadata)), // Propagate errors Value::Error { error, .. } => Err(*error), other => Err(ShellError::OnlySupportsThisInputType { @@ -176,13 +176,13 @@ fn values( match get_values(&vals, head, head) { Ok(cols) => Ok(cols .into_iter() - .into_pipeline_data_with_metadata(head, ctrlc, metadata)), + .into_pipeline_data_with_metadata(head, signals, metadata)), Err(err) => Err(err), } } PipelineData::ByteStream(stream, ..) => Err(ShellError::OnlySupportsThisInputType { exp_input_type: "record or table".into(), - wrong_type: "byte stream".into(), + wrong_type: stream.type_().describe().into(), dst_span: head, src_span: stream.span(), }), diff --git a/crates/nu-command/src/filters/where_.rs b/crates/nu-command/src/filters/where_.rs index fe73de354f..0a8c8c9ef3 100644 --- a/crates/nu-command/src/filters/where_.rs +++ b/crates/nu-command/src/filters/where_.rs @@ -1,5 +1,5 @@ use nu_engine::{command_prelude::*, ClosureEval}; -use nu_protocol::engine::Closure; +use nu_protocol::engine::{Closure, CommandType}; #[derive(Clone)] pub struct Where; @@ -19,6 +19,10 @@ tables, known as "row conditions". On the other hand, reading the condition from not supported."# } + fn command_type(&self) -> CommandType { + CommandType::Keyword + } + fn signature(&self) -> nu_protocol::Signature { Signature::build("where") .input_output_types(vec![ @@ -66,7 +70,7 @@ not supported."# Err(err) => Some(Value::error(err, head)), } }) - .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata(head, engine_state.signals().clone(), metadata)) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/filters/window.rs b/crates/nu-command/src/filters/window.rs index 5b386b9f84..3c470f478d 100644 --- a/crates/nu-command/src/filters/window.rs +++ b/crates/nu-command/src/filters/window.rs @@ -113,7 +113,6 @@ impl Command for Window { ) -> Result { let head = call.head; let group_size: Spanned = call.req(engine_state, stack, 0)?; - let ctrlc = engine_state.ctrlc.clone(); let metadata = input.metadata(); let stride: Option = call.get_flag(engine_state, stack, "stride")?; let remainder = call.has_flag(engine_state, stack, "remainder")?; @@ -131,7 +130,11 @@ impl Command for Window { remainder, }; - Ok(each_group_iterator.into_pipeline_data_with_metadata(head, ctrlc, metadata)) + Ok(each_group_iterator.into_pipeline_data_with_metadata( + head, + engine_state.signals().clone(), + metadata, + )) } } diff --git a/crates/nu-command/src/filters/wrap.rs b/crates/nu-command/src/filters/wrap.rs index 52a0fb22c3..dfe1c11d03 100644 --- a/crates/nu-command/src/filters/wrap.rs +++ b/crates/nu-command/src/filters/wrap.rs @@ -42,7 +42,7 @@ impl Command for Wrap { | PipelineData::ListStream { .. } => Ok(input .into_iter() .map(move |x| Value::record(record! { name.clone() => x }, span)) - .into_pipeline_data_with_metadata(span, engine_state.ctrlc.clone(), metadata)), + .into_pipeline_data_with_metadata(span, engine_state.signals().clone(), metadata)), PipelineData::ByteStream(stream, ..) => Ok(Value::record( record! { name => stream.into_value()? }, span, diff --git a/crates/nu-command/src/filters/zip.rs b/crates/nu-command/src/filters/zip.rs index 9d81451ed4..59e2b4ac98 100644 --- a/crates/nu-command/src/filters/zip.rs +++ b/crates/nu-command/src/filters/zip.rs @@ -112,7 +112,7 @@ impl Command for Zip { .into_iter() .zip(other) .map(move |(x, y)| Value::list(vec![x, y], head)) - .into_pipeline_data_with_metadata(head, engine_state.ctrlc.clone(), metadata)) + .into_pipeline_data_with_metadata(head, engine_state.signals().clone(), metadata)) } } diff --git a/crates/nu-command/src/formats/from/delimited.rs b/crates/nu-command/src/formats/from/delimited.rs index 1fdea19482..0fea7e082b 100644 --- a/crates/nu-command/src/formats/from/delimited.rs +++ b/crates/nu-command/src/formats/from/delimited.rs @@ -1,7 +1,14 @@ use csv::{ReaderBuilder, Trim}; -use nu_protocol::{IntoPipelineData, PipelineData, ShellError, Span, Value}; +use nu_protocol::{ByteStream, ListStream, PipelineData, ShellError, Signals, Span, Value}; -fn from_delimited_string_to_value( +fn from_csv_error(err: csv::Error, span: Span) -> ShellError { + ShellError::DelimiterError { + msg: err.to_string(), + span, + } +} + +fn from_delimited_stream( DelimitedReaderConfig { separator, comment, @@ -12,9 +19,15 @@ fn from_delimited_string_to_value( no_infer, trim, }: DelimitedReaderConfig, - s: String, + input: ByteStream, span: Span, -) -> Result { +) -> Result { + let input_reader = if let Some(stream) = input.reader() { + stream + } else { + return Ok(ListStream::new(std::iter::empty(), span, Signals::empty())); + }; + let mut reader = ReaderBuilder::new() .has_headers(!noheaders) .flexible(flexible) @@ -23,19 +36,29 @@ fn from_delimited_string_to_value( .quote(quote as u8) .escape(escape.map(|c| c as u8)) .trim(trim) - .from_reader(s.as_bytes()); + .from_reader(input_reader); let headers = if noheaders { - (1..=reader.headers()?.len()) + (0..reader + .headers() + .map_err(|err| from_csv_error(err, span))? + .len()) .map(|i| format!("column{i}")) .collect::>() } else { - reader.headers()?.iter().map(String::from).collect() + reader + .headers() + .map_err(|err| from_csv_error(err, span))? + .iter() + .map(String::from) + .collect() }; - let mut rows = vec![]; - for row in reader.records() { - let row = row?; + let iter = reader.into_records().map(move |row| { + let row = match row { + Ok(row) => row, + Err(err) => return Value::error(from_csv_error(err, span), span), + }; let columns = headers.iter().cloned(); let values = row .into_iter() @@ -57,10 +80,10 @@ fn from_delimited_string_to_value( // // Otherwise, if there are less values than headers, // then `Value::nothing(span)` is used to fill the remaining columns. - rows.push(Value::record(columns.zip(values).collect(), span)); - } + Value::record(columns.zip(values).collect(), span) + }); - Ok(Value::list(rows, span)) + Ok(ListStream::new(iter, span, Signals::empty())) } pub(super) struct DelimitedReaderConfig { @@ -79,14 +102,27 @@ pub(super) fn from_delimited_data( input: PipelineData, name: Span, ) -> Result { - let (concat_string, _span, metadata) = input.collect_string_strict(name)?; - - Ok(from_delimited_string_to_value(config, concat_string, name) - .map_err(|x| ShellError::DelimiterError { - msg: x.to_string(), - span: name, - })? - .into_pipeline_data_with_metadata(metadata)) + match input { + PipelineData::Empty => Ok(PipelineData::Empty), + PipelineData::Value(value, metadata) => { + let string = value.into_string()?; + let byte_stream = ByteStream::read_string(string, name, Signals::empty()); + Ok(PipelineData::ListStream( + from_delimited_stream(config, byte_stream, name)?, + metadata, + )) + } + PipelineData::ListStream(list_stream, _) => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "string".into(), + wrong_type: "list".into(), + dst_span: name, + src_span: list_stream.span(), + }), + PipelineData::ByteStream(byte_stream, metadata) => Ok(PipelineData::ListStream( + from_delimited_stream(config, byte_stream, name)?, + metadata, + )), + } } pub fn trim_from_str(trim: Option) -> Result { diff --git a/crates/nu-command/src/formats/from/json.rs b/crates/nu-command/src/formats/from/json.rs index ea449711c1..6252afff4c 100644 --- a/crates/nu-command/src/formats/from/json.rs +++ b/crates/nu-command/src/formats/from/json.rs @@ -1,4 +1,7 @@ +use std::io::{BufRead, Cursor}; + use nu_engine::command_prelude::*; +use nu_protocol::{ListStream, PipelineMetadata, Signals}; #[derive(Clone)] pub struct FromJson; @@ -45,6 +48,15 @@ impl Command for FromJson { "b" => Value::test_int(2), })), }, + Example { + example: r#"'{ "a": 1 } +{ "b": 2 }' | from json --objects"#, + description: "Parse a stream of line-delimited JSON values", + result: Some(Value::test_list(vec![ + Value::test_record(record! {"a" => Value::test_int(1)}), + Value::test_record(record! {"b" => Value::test_int(2)}), + ])), + }, ] } @@ -56,49 +68,85 @@ impl Command for FromJson { input: PipelineData, ) -> Result { let span = call.head; - let (string_input, span, metadata) = input.collect_string_strict(span)?; - - if string_input.is_empty() { - return Ok(Value::nothing(span).into_pipeline_data()); - } let strict = call.has_flag(engine_state, stack, "strict")?; // TODO: turn this into a structured underline of the nu_json error if call.has_flag(engine_state, stack, "objects")? { - let lines = string_input.lines().filter(|line| !line.trim().is_empty()); - - let converted_lines: Vec<_> = if strict { - lines - .map(|line| { - convert_string_to_value_strict(line, span) - .unwrap_or_else(|err| Value::error(err, span)) - }) - .collect() - } else { - lines - .map(|line| { - convert_string_to_value(line, span) - .unwrap_or_else(|err| Value::error(err, span)) - }) - .collect() - }; - - Ok(converted_lines.into_pipeline_data_with_metadata( - span, - engine_state.ctrlc.clone(), - metadata, - )) - } else if strict { - Ok(convert_string_to_value_strict(&string_input, span)? - .into_pipeline_data_with_metadata(metadata)) + // Return a stream of JSON values, one for each non-empty line + match input { + PipelineData::Value(Value::String { val, .. }, metadata) => { + Ok(PipelineData::ListStream( + read_json_lines( + Cursor::new(val), + span, + strict, + engine_state.signals().clone(), + ), + update_metadata(metadata), + )) + } + PipelineData::ByteStream(stream, metadata) + if stream.type_() != ByteStreamType::Binary => + { + if let Some(reader) = stream.reader() { + Ok(PipelineData::ListStream( + read_json_lines(reader, span, strict, Signals::empty()), + update_metadata(metadata), + )) + } else { + Ok(PipelineData::Empty) + } + } + _ => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "string".into(), + wrong_type: input.get_type().to_string(), + dst_span: call.head, + src_span: input.span().unwrap_or(call.head), + }), + } } else { - Ok(convert_string_to_value(&string_input, span)? - .into_pipeline_data_with_metadata(metadata)) + // Return a single JSON value + let (string_input, span, metadata) = input.collect_string_strict(span)?; + + if string_input.is_empty() { + return Ok(Value::nothing(span).into_pipeline_data()); + } + + if strict { + Ok(convert_string_to_value_strict(&string_input, span)? + .into_pipeline_data_with_metadata(update_metadata(metadata))) + } else { + Ok(convert_string_to_value(&string_input, span)? + .into_pipeline_data_with_metadata(update_metadata(metadata))) + } } } } +/// Create a stream of values from a reader that produces line-delimited JSON +fn read_json_lines( + input: impl BufRead + Send + 'static, + span: Span, + strict: bool, + signals: Signals, +) -> ListStream { + let iter = input + .lines() + .filter(|line| line.as_ref().is_ok_and(|line| !line.trim().is_empty()) || line.is_err()) + .map(move |line| { + let line = line.err_span(span)?; + if strict { + convert_string_to_value_strict(&line, span) + } else { + convert_string_to_value(&line, span) + } + }) + .map(move |result| result.unwrap_or_else(|err| Value::error(err, span))); + + ListStream::new(iter, span, signals) +} + fn convert_nujson_to_value(value: nu_json::Value, span: Span) -> Value { match value { nu_json::Value::Array(array) => Value::list( @@ -217,6 +265,14 @@ fn convert_string_to_value_strict(string_input: &str, span: Span) -> Result) -> Option { + metadata + .map(|md| md.with_content_type(Some("application/json".into()))) + .or_else(|| { + Some(PipelineMetadata::default().with_content_type(Some("application/json".into()))) + }) +} + #[cfg(test)] mod test { use super::*; diff --git a/crates/nu-command/src/formats/from/msgpack.rs b/crates/nu-command/src/formats/from/msgpack.rs index 4d8ea5e320..d3d0d710f1 100644 --- a/crates/nu-command/src/formats/from/msgpack.rs +++ b/crates/nu-command/src/formats/from/msgpack.rs @@ -5,12 +5,12 @@ use std::{ error::Error, io::{self, Cursor, ErrorKind}, string::FromUtf8Error, - sync::{atomic::AtomicBool, Arc}, }; use byteorder::{BigEndian, ReadBytesExt}; use chrono::{TimeZone, Utc}; use nu_engine::command_prelude::*; +use nu_protocol::Signals; use rmp::decode::{self as mp, ValueReadError}; /// Max recursion depth @@ -111,7 +111,7 @@ MessagePack: https://msgpack.org/ let opts = Opts { span: call.head, objects, - ctrlc: engine_state.ctrlc.clone(), + signals: engine_state.signals().clone(), }; match input { // Deserialize from a byte buffer @@ -227,7 +227,7 @@ impl From for ShellError { pub(crate) struct Opts { pub span: Span, pub objects: bool, - pub ctrlc: Option>, + pub signals: Signals, } /// Read single or multiple values into PipelineData @@ -238,7 +238,7 @@ pub(crate) fn read_msgpack( let Opts { span, objects, - ctrlc, + signals, } = opts; if objects { // Make an iterator that reads multiple values from the reader @@ -262,7 +262,7 @@ pub(crate) fn read_msgpack( None } }) - .into_pipeline_data(span, ctrlc)) + .into_pipeline_data(span, signals)) } else { // Read a single value and then make sure it's EOF let result = read_value(&mut input, span, 0)?; diff --git a/crates/nu-command/src/formats/from/msgpackz.rs b/crates/nu-command/src/formats/from/msgpackz.rs index 7960f3f97a..c98b72cdd6 100644 --- a/crates/nu-command/src/formats/from/msgpackz.rs +++ b/crates/nu-command/src/formats/from/msgpackz.rs @@ -41,7 +41,7 @@ impl Command for FromMsgpackz { let opts = Opts { span, objects, - ctrlc: engine_state.ctrlc.clone(), + signals: engine_state.signals().clone(), }; match input { // Deserialize from a byte buffer diff --git a/crates/nu-command/src/formats/from/ssv.rs b/crates/nu-command/src/formats/from/ssv.rs index 5efb2a6c3b..1d17dfc69d 100644 --- a/crates/nu-command/src/formats/from/ssv.rs +++ b/crates/nu-command/src/formats/from/ssv.rs @@ -52,12 +52,12 @@ impl Command for FromSsv { Value::test_list( vec![ Value::test_record(record! { - "column1" => Value::test_string("FOO"), - "column2" => Value::test_string("BAR"), + "column0" => Value::test_string("FOO"), + "column1" => Value::test_string("BAR"), }), Value::test_record(record! { - "column1" => Value::test_string("1"), - "column2" => Value::test_string("2"), + "column0" => Value::test_string("1"), + "column1" => Value::test_string("2"), }), ], ) @@ -170,7 +170,7 @@ fn parse_aligned_columns<'a>( let headers: Vec<(String, usize)> = indices .iter() .enumerate() - .map(|(i, position)| (format!("column{}", i + 1), *position)) + .map(|(i, position)| (format!("column{}", i), *position)) .collect(); construct(ls.iter().map(|s| s.to_owned()), headers) @@ -215,7 +215,7 @@ fn parse_separated_columns<'a>( let parse_without_headers = |ls: Vec<&str>| { let num_columns = ls.iter().map(|r| r.len()).max().unwrap_or(0); - let headers = (1..=num_columns) + let headers = (0..=num_columns) .map(|i| format!("column{i}")) .collect::>(); collect(headers, ls.into_iter(), separator) @@ -370,9 +370,9 @@ mod tests { assert_eq!( result, vec![ - vec![owned("column1", "a"), owned("column2", "b")], - vec![owned("column1", "1"), owned("column2", "2")], - vec![owned("column1", "3"), owned("column2", "4")] + vec![owned("column0", "a"), owned("column1", "b")], + vec![owned("column0", "1"), owned("column1", "2")], + vec![owned("column0", "3"), owned("column1", "4")] ] ); } @@ -484,25 +484,25 @@ mod tests { result, vec![ vec![ - owned("column1", "a multi-word value"), - owned("column2", "b"), - owned("column3", ""), - owned("column4", "d"), - owned("column5", "") - ], - vec![ - owned("column1", "1"), + owned("column0", "a multi-word value"), + owned("column1", "b"), owned("column2", ""), - owned("column3", "3-3"), - owned("column4", "4"), - owned("column5", "") + owned("column3", "d"), + owned("column4", "") ], vec![ + owned("column0", "1"), + owned("column1", ""), + owned("column2", "3-3"), + owned("column3", "4"), + owned("column4", "") + ], + vec![ + owned("column0", ""), owned("column1", ""), owned("column2", ""), owned("column3", ""), - owned("column4", ""), - owned("column5", "last") + owned("column4", "last") ], ] ); diff --git a/crates/nu-command/src/formats/from/toml.rs b/crates/nu-command/src/formats/from/toml.rs index e1ddca3164..266911f50a 100644 --- a/crates/nu-command/src/formats/from/toml.rs +++ b/crates/nu-command/src/formats/from/toml.rs @@ -1,5 +1,5 @@ use nu_engine::command_prelude::*; -use std::str::FromStr; +use toml::value::{Datetime, Offset}; #[derive(Clone)] pub struct FromToml; @@ -56,6 +56,54 @@ b = [1, 2]' | from toml", } } +fn convert_toml_datetime_to_value(dt: &Datetime, span: Span) -> Value { + match &dt.clone() { + toml::value::Datetime { + date: Some(_), + time: _, + offset: _, + } => (), + _ => return Value::string(dt.to_string(), span), + } + + let date = match dt.date { + Some(date) => { + chrono::NaiveDate::from_ymd_opt(date.year.into(), date.month.into(), date.day.into()) + } + None => Some(chrono::NaiveDate::default()), + }; + + let time = match dt.time { + Some(time) => chrono::NaiveTime::from_hms_nano_opt( + time.hour.into(), + time.minute.into(), + time.second.into(), + time.nanosecond, + ), + None => Some(chrono::NaiveTime::default()), + }; + + let tz = match dt.offset { + Some(offset) => match offset { + Offset::Z => chrono::FixedOffset::east_opt(0), + Offset::Custom { minutes: min } => chrono::FixedOffset::east_opt(min as i32 * 60), + }, + None => chrono::FixedOffset::east_opt(0), + }; + + let datetime = match (date, time, tz) { + (Some(date), Some(time), Some(tz)) => chrono::NaiveDateTime::new(date, time) + .and_local_timezone(tz) + .earliest(), + _ => None, + }; + + match datetime { + Some(datetime) => Value::date(datetime, span), + None => Value::string(dt.to_string(), span), + } +} + fn convert_toml_to_value(value: &toml::Value, span: Span) -> Value { match value { toml::Value::Array(array) => { @@ -76,13 +124,7 @@ fn convert_toml_to_value(value: &toml::Value, span: Span) -> Value { span, ), toml::Value::String(s) => Value::string(s.clone(), span), - toml::Value::Datetime(d) => match chrono::DateTime::from_str(&d.to_string()) { - Ok(nushell_date) => Value::date(nushell_date, span), - // in the unlikely event that parsing goes wrong, this function still returns a valid - // nushell date (however the default one). This decision was made to make the output of - // this function uniform amongst all eventualities - Err(_) => Value::date(chrono::DateTime::default(), span), - }, + toml::Value::Datetime(dt) => convert_toml_datetime_to_value(dt, span), } } @@ -113,32 +155,6 @@ mod tests { test_examples(FromToml {}) } - #[test] - fn from_toml_creates_nushell_date() { - let toml_date = toml::Value::Datetime(Datetime { - date: Option::from(toml::value::Date { - year: 1980, - month: 10, - day: 12, - }), - time: Option::from(toml::value::Time { - hour: 10, - minute: 12, - second: 44, - nanosecond: 0, - }), - offset: Option::from(toml::value::Offset::Custom { minutes: 120 }), - }); - - let span = Span::test_data(); - let reference_date = Value::date(Default::default(), Span::test_data()); - - let result = convert_toml_to_value(&toml_date, span); - - //positive test (from toml returns a nushell date) - assert_eq!(result.get_type(), reference_date.get_type()); - } - #[test] fn from_toml_creates_correct_date() { let toml_date = toml::Value::Datetime(Datetime { @@ -206,4 +222,113 @@ mod tests { assert!(result.is_err()); } + + #[test] + fn convert_toml_datetime_to_value_date_time_offset() { + let toml_date = Datetime { + date: Option::from(toml::value::Date { + year: 2000, + month: 1, + day: 1, + }), + time: Option::from(toml::value::Time { + hour: 12, + minute: 12, + second: 12, + nanosecond: 0, + }), + offset: Option::from(toml::value::Offset::Custom { minutes: 120 }), + }; + + let span = Span::test_data(); + let reference_date = Value::date( + chrono::FixedOffset::east_opt(60 * 120) + .unwrap() + .with_ymd_and_hms(2000, 1, 1, 12, 12, 12) + .unwrap(), + span, + ); + + let result = convert_toml_datetime_to_value(&toml_date, span); + + assert_eq!(result, reference_date); + } + + #[test] + fn convert_toml_datetime_to_value_date_time() { + let toml_date = Datetime { + date: Option::from(toml::value::Date { + year: 2000, + month: 1, + day: 1, + }), + time: Option::from(toml::value::Time { + hour: 12, + minute: 12, + second: 12, + nanosecond: 0, + }), + offset: None, + }; + + let span = Span::test_data(); + let reference_date = Value::date( + chrono::FixedOffset::east_opt(0) + .unwrap() + .with_ymd_and_hms(2000, 1, 1, 12, 12, 12) + .unwrap(), + span, + ); + + let result = convert_toml_datetime_to_value(&toml_date, span); + + assert_eq!(result, reference_date); + } + + #[test] + fn convert_toml_datetime_to_value_date() { + let toml_date = Datetime { + date: Option::from(toml::value::Date { + year: 2000, + month: 1, + day: 1, + }), + time: None, + offset: None, + }; + + let span = Span::test_data(); + let reference_date = Value::date( + chrono::FixedOffset::east_opt(0) + .unwrap() + .with_ymd_and_hms(2000, 1, 1, 0, 0, 0) + .unwrap(), + span, + ); + + let result = convert_toml_datetime_to_value(&toml_date, span); + + assert_eq!(result, reference_date); + } + + #[test] + fn convert_toml_datetime_to_value_only_time() { + let toml_date = Datetime { + date: None, + time: Option::from(toml::value::Time { + hour: 12, + minute: 12, + second: 12, + nanosecond: 0, + }), + offset: None, + }; + + let span = Span::test_data(); + let reference_date = Value::string(toml_date.to_string(), span); + + let result = convert_toml_datetime_to_value(&toml_date, span); + + assert_eq!(result, reference_date); + } } diff --git a/crates/nu-command/src/formats/to/csv.rs b/crates/nu-command/src/formats/to/csv.rs index 173c6fbd6b..e7786e60cc 100644 --- a/crates/nu-command/src/formats/to/csv.rs +++ b/crates/nu-command/src/formats/to/csv.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use crate::formats::to::delimited::to_delimited_data; use nu_engine::command_prelude::*; use nu_protocol::Config; @@ -27,26 +29,37 @@ impl Command for ToCsv { "do not output the columns names as the first row", Some('n'), ) + .named( + "columns", + SyntaxShape::List(SyntaxShape::String.into()), + "the names (in order) of the columns to use", + None, + ) .category(Category::Formats) } fn examples(&self) -> Vec { vec![ Example { - description: "Outputs an CSV string representing the contents of this table", + description: "Outputs a CSV string representing the contents of this table", example: "[[foo bar]; [1 2]] | to csv", result: Some(Value::test_string("foo,bar\n1,2\n")), }, Example { - description: "Outputs an CSV string representing the contents of this table", + description: "Outputs a CSV string representing the contents of this table", example: "[[foo bar]; [1 2]] | to csv --separator ';' ", result: Some(Value::test_string("foo;bar\n1;2\n")), }, Example { - description: "Outputs an CSV string representing the contents of this record", + description: "Outputs a CSV string representing the contents of this record", example: "{a: 1 b: 2} | to csv", result: Some(Value::test_string("a,b\n1,2\n")), }, + Example { + description: "Outputs a CSV stream with column names pre-determined", + example: "[[foo bar baz]; [1 2 3]] | to csv --columns [baz foo]", + result: Some(Value::test_string("baz,foo\n3,1\n")), + }, ] } @@ -64,8 +77,9 @@ impl Command for ToCsv { let head = call.head; let noheaders = call.has_flag(engine_state, stack, "noheaders")?; let separator: Option> = call.get_flag(engine_state, stack, "separator")?; - let config = engine_state.get_config(); - to_csv(input, noheaders, separator, head, config) + let columns: Option> = call.get_flag(engine_state, stack, "columns")?; + let config = engine_state.config.clone(); + to_csv(input, noheaders, separator, columns, head, config) } } @@ -73,13 +87,14 @@ fn to_csv( input: PipelineData, noheaders: bool, separator: Option>, + columns: Option>, head: Span, - config: &Config, + config: Arc, ) -> Result { let sep = match separator { Some(Spanned { item: s, span, .. }) => { if s == r"\t" { - '\t' + Spanned { item: '\t', span } } else { let vec_s: Vec = s.chars().collect(); if vec_s.len() != 1 { @@ -89,13 +104,19 @@ fn to_csv( span, }); }; - vec_s[0] + Spanned { + item: vec_s[0], + span: head, + } } } - _ => ',', + _ => Spanned { + item: ',', + span: head, + }, }; - to_delimited_data(noheaders, sep, "CSV", input, head, config) + to_delimited_data(noheaders, sep, columns, "CSV", input, head, config) } #[cfg(test)] diff --git a/crates/nu-command/src/formats/to/delimited.rs b/crates/nu-command/src/formats/to/delimited.rs index 490983d67b..34bb06b8a2 100644 --- a/crates/nu-command/src/formats/to/delimited.rs +++ b/crates/nu-command/src/formats/to/delimited.rs @@ -1,113 +1,31 @@ -use csv::{Writer, WriterBuilder}; +use csv::WriterBuilder; use nu_cmd_base::formats::to::delimited::merge_descriptors; -use nu_protocol::{Config, IntoPipelineData, PipelineData, Record, ShellError, Span, Value}; -use std::{collections::VecDeque, error::Error}; +use nu_protocol::{ + ByteStream, ByteStreamType, Config, PipelineData, ShellError, Signals, Span, Spanned, Value, +}; +use std::{iter, sync::Arc}; -fn from_value_to_delimited_string( - value: &Value, - separator: char, - config: &Config, - head: Span, -) -> Result { - let span = value.span(); - match value { - Value::Record { val, .. } => record_to_delimited(val, span, separator, config, head), - Value::List { vals, .. } => table_to_delimited(vals, span, separator, config, head), - // Propagate errors by explicitly matching them before the final case. - Value::Error { error, .. } => Err(*error.clone()), - v => Err(make_unsupported_input_error(v, head, v.span())), - } -} - -fn record_to_delimited( - record: &Record, - span: Span, - separator: char, - config: &Config, - head: Span, -) -> Result { - let mut wtr = WriterBuilder::new() - .delimiter(separator as u8) - .from_writer(vec![]); - let mut fields: VecDeque = VecDeque::new(); - let mut values: VecDeque = VecDeque::new(); - - for (k, v) in record { - fields.push_back(k.clone()); - - values.push_back(to_string_tagged_value(v, config, head, span)?); - } - - wtr.write_record(fields).expect("can not write."); - wtr.write_record(values).expect("can not write."); - - writer_to_string(wtr).map_err(|_| make_conversion_error("record", span)) -} - -fn table_to_delimited( - vals: &[Value], - span: Span, - separator: char, - config: &Config, - head: Span, -) -> Result { - if let Some(val) = find_non_record(vals) { - return Err(make_unsupported_input_error(val, head, span)); - } - - let mut wtr = WriterBuilder::new() - .delimiter(separator as u8) - .from_writer(vec![]); - - let merged_descriptors = merge_descriptors(vals); - - if merged_descriptors.is_empty() { - let vals = vals - .iter() - .map(|ele| { - to_string_tagged_value(ele, config, head, span).unwrap_or_else(|_| String::new()) - }) - .collect::>(); - wtr.write_record(vals).expect("can not write"); - } else { - wtr.write_record(merged_descriptors.iter().map(|item| &item[..])) - .expect("can not write."); - - for l in vals { - // should always be true because of `find_non_record` above - if let Value::Record { val: l, .. } = l { - let mut row = vec![]; - for desc in &merged_descriptors { - row.push(match l.get(desc) { - Some(s) => to_string_tagged_value(s, config, head, span)?, - None => String::new(), - }); - } - wtr.write_record(&row).expect("can not write"); - } +fn make_csv_error(error: csv::Error, format_name: &str, head: Span) -> ShellError { + if let csv::ErrorKind::Io(error) = error.kind() { + ShellError::IOErrorSpanned { + msg: error.to_string(), + span: head, + } + } else { + ShellError::GenericError { + error: format!("Failed to generate {format_name} data"), + msg: error.to_string(), + span: Some(head), + help: None, + inner: vec![], } - } - writer_to_string(wtr).map_err(|_| make_conversion_error("table", span)) -} - -fn writer_to_string(writer: Writer>) -> Result> { - Ok(String::from_utf8(writer.into_inner()?)?) -} - -fn make_conversion_error(type_from: &str, span: Span) -> ShellError { - ShellError::CantConvert { - to_type: type_from.to_string(), - from_type: "string".to_string(), - span, - help: None, } } fn to_string_tagged_value( v: &Value, config: &Config, - span: Span, - head: Span, + format_name: &'static str, ) -> Result { match &v { Value::String { .. } @@ -123,50 +41,129 @@ fn to_string_tagged_value( Value::Nothing { .. } => Ok(String::new()), // Propagate existing errors Value::Error { error, .. } => Err(*error.clone()), - _ => Err(make_unsupported_input_error(v, head, span)), + _ => Err(make_cant_convert_error(v, format_name)), } } -fn make_unsupported_input_error(value: &Value, head: Span, span: Span) -> ShellError { +fn make_unsupported_input_error( + r#type: impl std::fmt::Display, + head: Span, + span: Span, +) -> ShellError { ShellError::UnsupportedInput { - msg: "Unexpected type".to_string(), - input: format!("input type: {:?}", value.get_type()), + msg: "expected table or record".to_string(), + input: format!("input type: {}", r#type), msg_span: head, input_span: span, } } -pub fn find_non_record(values: &[Value]) -> Option<&Value> { - values - .iter() - .find(|val| !matches!(val, Value::Record { .. })) +fn make_cant_convert_error(value: &Value, format_name: &'static str) -> ShellError { + ShellError::CantConvert { + to_type: "string".into(), + from_type: value.get_type().to_string(), + span: value.span(), + help: Some(format!( + "only simple values are supported for {format_name} output" + )), + } } pub fn to_delimited_data( noheaders: bool, - sep: char, + separator: Spanned, + columns: Option>, format_name: &'static str, input: PipelineData, - span: Span, - config: &Config, + head: Span, + config: Arc, ) -> Result { - let value = input.into_value(span)?; - let output = match from_value_to_delimited_string(&value, sep, config, span) { - Ok(mut x) => { - if noheaders { - if let Some(second_line) = x.find('\n') { - let start = second_line + 1; - x.replace_range(0..start, ""); - } - } - Ok(x) + let mut input = input; + let span = input.span().unwrap_or(head); + let metadata = input.metadata(); + + let separator = u8::try_from(separator.item).map_err(|_| ShellError::IncorrectValue { + msg: "separator must be an ASCII character".into(), + val_span: separator.span, + call_span: head, + })?; + + // Check to ensure the input is likely one of our supported types first. We can't check a stream + // without consuming it though + match input { + PipelineData::Value(Value::List { .. } | Value::Record { .. }, _) => (), + PipelineData::Value(Value::Error { error, .. }, _) => return Err(*error), + PipelineData::Value(other, _) => { + return Err(make_unsupported_input_error(other.get_type(), head, span)) } - Err(_) => Err(ShellError::CantConvert { - to_type: format_name.into(), - from_type: value.get_type().to_string(), - span: value.span(), - help: None, - }), - }?; - Ok(Value::string(output, span).into_pipeline_data()) + PipelineData::ByteStream(..) => { + return Err(make_unsupported_input_error("byte stream", head, span)) + } + PipelineData::ListStream(..) => (), + PipelineData::Empty => (), + } + + // Determine the columns we'll use. This is necessary even if we don't write the header row, + // because we need to write consistent columns. + let columns = match columns { + Some(columns) => columns, + None => { + // The columns were not provided. We need to detect them, and in order to do so, we have + // to convert the input into a value first, so that we can find all of them + let value = input.into_value(span)?; + let columns = match &value { + Value::List { vals, .. } => merge_descriptors(vals), + Value::Record { val, .. } => val.columns().cloned().collect(), + _ => return Err(make_unsupported_input_error(value.get_type(), head, span)), + }; + input = PipelineData::Value(value, metadata.clone()); + columns + } + }; + + // Generate a byte stream of all of the values in the pipeline iterator, with a non-strict + // iterator so we can still accept plain records. + let mut iter = input.into_iter(); + + // If we're configured to generate a header, we generate it first, then set this false + let mut is_header = !noheaders; + + let stream = ByteStream::from_fn( + head, + Signals::empty(), + ByteStreamType::String, + move |buffer| { + let mut wtr = WriterBuilder::new() + .delimiter(separator) + .from_writer(buffer); + + if is_header { + // Unless we are configured not to write a header, we write the header row now, once, + // before everything else. + wtr.write_record(&columns) + .map_err(|err| make_csv_error(err, format_name, head))?; + is_header = false; + Ok(true) + } else if let Some(row) = iter.next() { + // Write each column of a normal row, in order + let record = row.into_record()?; + for column in &columns { + let field = record + .get(column) + .map(|v| to_string_tagged_value(v, &config, format_name)) + .unwrap_or(Ok(String::new()))?; + wtr.write_field(field) + .map_err(|err| make_csv_error(err, format_name, head))?; + } + // End the row + wtr.write_record(iter::empty::()) + .map_err(|err| make_csv_error(err, format_name, head))?; + Ok(true) + } else { + Ok(false) + } + }, + ); + + Ok(PipelineData::ByteStream(stream, metadata)) } diff --git a/crates/nu-command/src/formats/to/json.rs b/crates/nu-command/src/formats/to/json.rs index c4c87f804f..5722387e2c 100644 --- a/crates/nu-command/src/formats/to/json.rs +++ b/crates/nu-command/src/formats/to/json.rs @@ -1,5 +1,5 @@ use nu_engine::command_prelude::*; -use nu_protocol::ast::PathMember; +use nu_protocol::{ast::PathMember, PipelineMetadata}; #[derive(Clone)] pub struct ToJson; @@ -61,7 +61,12 @@ impl Command for ToJson { match json_result { Ok(serde_json_string) => { - Ok(Value::string(serde_json_string, span).into_pipeline_data()) + let res = Value::string(serde_json_string, span); + let metadata = PipelineMetadata { + data_source: nu_protocol::DataSource::None, + content_type: Some("application/json".to_string()), + }; + Ok(PipelineData::Value(res, Some(metadata))) } _ => Ok(Value::error( ShellError::CantConvert { diff --git a/crates/nu-command/src/formats/to/msgpack.rs b/crates/nu-command/src/formats/to/msgpack.rs index bfeb428e3e..02515e26fe 100644 --- a/crates/nu-command/src/formats/to/msgpack.rs +++ b/crates/nu-command/src/formats/to/msgpack.rs @@ -5,7 +5,7 @@ use std::io; use byteorder::{BigEndian, WriteBytesExt}; use nu_engine::command_prelude::*; -use nu_protocol::{ast::PathMember, Spanned}; +use nu_protocol::{ast::PathMember, Signals, Spanned}; use rmp::encode as mp; /// Max recursion depth @@ -189,7 +189,7 @@ pub(crate) fn write_value( // Convert range to list write_value( out, - &Value::list(val.into_range_iter(span, None).collect(), span), + &Value::list(val.into_range_iter(span, Signals::empty()).collect(), span), depth, )?; } diff --git a/crates/nu-command/src/formats/to/msgpackz.rs b/crates/nu-command/src/formats/to/msgpackz.rs index 9168d05018..83d812cab6 100644 --- a/crates/nu-command/src/formats/to/msgpackz.rs +++ b/crates/nu-command/src/formats/to/msgpackz.rs @@ -5,7 +5,7 @@ use nu_engine::command_prelude::*; use super::msgpack::write_value; const BUFFER_SIZE: usize = 65536; -const DEFAULT_QUALITY: u32 = 1; +const DEFAULT_QUALITY: u32 = 3; // 1 can be very bad const DEFAULT_WINDOW_SIZE: u32 = 20; #[derive(Clone)] @@ -22,7 +22,7 @@ impl Command for ToMsgpackz { .named( "quality", SyntaxShape::Int, - "Quality of brotli compression (default 1)", + "Quality of brotli compression (default 3)", Some('q'), ) .named( diff --git a/crates/nu-command/src/formats/to/text.rs b/crates/nu-command/src/formats/to/text.rs index 7f1d632c13..82b853248c 100644 --- a/crates/nu-command/src/formats/to/text.rs +++ b/crates/nu-command/src/formats/to/text.rs @@ -1,6 +1,8 @@ use chrono_humanize::HumanTime; use nu_engine::command_prelude::*; -use nu_protocol::{format_duration, format_filesize_from_conf, ByteStream, Config}; +use nu_protocol::{ + format_duration, format_filesize_from_conf, ByteStream, Config, PipelineMetadata, +}; const LINE_ENDING: &str = if cfg!(target_os = "windows") { "\r\n" @@ -37,10 +39,14 @@ impl Command for ToText { let input = input.try_expand_range()?; match input { - PipelineData::Empty => Ok(Value::string(String::new(), span).into_pipeline_data()), + PipelineData::Empty => Ok(Value::string(String::new(), span) + .into_pipeline_data_with_metadata(update_metadata(None))), PipelineData::Value(value, ..) => { let str = local_into_string(value, LINE_ENDING, engine_state.get_config()); - Ok(Value::string(str, span).into_pipeline_data()) + Ok( + Value::string(str, span) + .into_pipeline_data_with_metadata(update_metadata(None)), + ) } PipelineData::ListStream(stream, meta) => { let span = stream.span(); @@ -51,11 +57,18 @@ impl Command for ToText { str }); Ok(PipelineData::ByteStream( - ByteStream::from_iter(iter, span, engine_state.ctrlc.clone()), - meta, + ByteStream::from_iter( + iter, + span, + engine_state.signals().clone(), + ByteStreamType::String, + ), + update_metadata(meta), )) } - PipelineData::ByteStream(stream, meta) => Ok(PipelineData::ByteStream(stream, meta)), + PipelineData::ByteStream(stream, meta) => { + Ok(PipelineData::ByteStream(stream, update_metadata(meta))) + } } } @@ -119,6 +132,14 @@ fn local_into_string(value: Value, separator: &str, config: &Config) -> String { } } +fn update_metadata(metadata: Option) -> Option { + metadata + .map(|md| md.with_content_type(Some("text/plain".to_string()))) + .or_else(|| { + Some(PipelineMetadata::default().with_content_type(Some("text/plain".to_string()))) + }) +} + #[cfg(test)] mod test { use super::*; diff --git a/crates/nu-command/src/formats/to/toml.rs b/crates/nu-command/src/formats/to/toml.rs index 7423f147dd..ed1490231c 100644 --- a/crates/nu-command/src/formats/to/toml.rs +++ b/crates/nu-command/src/formats/to/toml.rs @@ -1,4 +1,4 @@ -use chrono::SecondsFormat; +use chrono::{DateTime, Datelike, FixedOffset, Timelike}; use nu_engine::command_prelude::*; use nu_protocol::ast::PathMember; @@ -24,7 +24,7 @@ impl Command for ToToml { vec![Example { description: "Outputs an TOML string representing the contents of this record", example: r#"{foo: 1 bar: 'qwe'} | to toml"#, - result: Some(Value::test_string("bar = \"qwe\"\nfoo = 1\n")), + result: Some(Value::test_string("foo = 1\nbar = \"qwe\"\n")), }] } @@ -49,9 +49,7 @@ fn helper(engine_state: &EngineState, v: &Value) -> Result toml::Value::Integer(*val), Value::Filesize { val, .. } => toml::Value::Integer(*val), Value::Duration { val, .. } => toml::Value::String(val.to_string()), - Value::Date { val, .. } => { - toml::Value::String(val.to_rfc3339_opts(SecondsFormat::AutoSi, false)) - } + Value::Date { val, .. } => toml::Value::Datetime(to_toml_datetime(val)), Value::Range { .. } => toml::Value::String("".to_string()), Value::Float { val, .. } => toml::Value::Float(*val), Value::String { val, .. } | Value::Glob { val, .. } => toml::Value::String(val.clone()), @@ -103,7 +101,7 @@ fn toml_into_pipeline_data( value_type: Type, span: Span, ) -> Result { - match toml::to_string(&toml_value) { + match toml::to_string_pretty(&toml_value) { Ok(serde_toml_string) => Ok(Value::string(serde_toml_string, span).into_pipeline_data()), _ => Ok(Value::error( ShellError::CantConvert { @@ -157,6 +155,43 @@ fn to_toml( } } +/// Convert chrono datetime into a toml::Value datetime. The latter uses its +/// own ad-hoc datetime types, which makes this somewhat convoluted. +fn to_toml_datetime(datetime: &DateTime) -> toml::value::Datetime { + let date = toml::value::Date { + // TODO: figure out what to do with BC dates, because the toml + // crate doesn't support them. Same for large years, which + // don't fit in u16. + year: datetime.year_ce().1 as u16, + // Panic: this is safe, because chrono guarantees that the month + // value will be between 1 and 12 and the day will be between 1 + // and 31 + month: datetime.month() as u8, + day: datetime.day() as u8, + }; + + let time = toml::value::Time { + // Panic: same as before, chorono guarantees that all of the following 3 + // methods return values less than 65'000 + hour: datetime.hour() as u8, + minute: datetime.minute() as u8, + second: datetime.second() as u8, + nanosecond: datetime.nanosecond(), + }; + + let offset = toml::value::Offset::Custom { + // Panic: minute timezone offset fits into i16 (that's more than + // 1000 hours) + minutes: (-datetime.timezone().utc_minus_local() / 60) as i16, + }; + + toml::value::Datetime { + date: Some(date), + time: Some(time), + offset: Some(offset), + } +} + #[cfg(test)] mod tests { use super::*; @@ -181,7 +216,20 @@ mod tests { Span::test_data(), ); - let reference_date = toml::Value::String(String::from("1980-10-12T10:12:44+02:00")); + let reference_date = toml::Value::Datetime(toml::value::Datetime { + date: Some(toml::value::Date { + year: 1980, + month: 10, + day: 12, + }), + time: Some(toml::value::Time { + hour: 10, + minute: 12, + second: 44, + nanosecond: 0, + }), + offset: Some(toml::value::Offset::Custom { minutes: 120 }), + }); let result = helper(&engine_state, &test_date); diff --git a/crates/nu-command/src/formats/to/tsv.rs b/crates/nu-command/src/formats/to/tsv.rs index eeaeb6b401..edc9592409 100644 --- a/crates/nu-command/src/formats/to/tsv.rs +++ b/crates/nu-command/src/formats/to/tsv.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use crate::formats::to::delimited::to_delimited_data; use nu_engine::command_prelude::*; use nu_protocol::Config; @@ -21,6 +23,12 @@ impl Command for ToTsv { "do not output the column names as the first row", Some('n'), ) + .named( + "columns", + SyntaxShape::List(SyntaxShape::String.into()), + "the names (in order) of the columns to use", + None, + ) .category(Category::Formats) } @@ -31,15 +39,20 @@ impl Command for ToTsv { fn examples(&self) -> Vec { vec![ Example { - description: "Outputs an TSV string representing the contents of this table", + description: "Outputs a TSV string representing the contents of this table", example: "[[foo bar]; [1 2]] | to tsv", result: Some(Value::test_string("foo\tbar\n1\t2\n")), }, Example { - description: "Outputs an TSV string representing the contents of this record", + description: "Outputs a TSV string representing the contents of this record", example: "{a: 1 b: 2} | to tsv", result: Some(Value::test_string("a\tb\n1\t2\n")), }, + Example { + description: "Outputs a TSV stream with column names pre-determined", + example: "[[foo bar baz]; [1 2 3]] | to tsv --columns [baz foo]", + result: Some(Value::test_string("baz\tfoo\n3\t1\n")), + }, ] } @@ -52,18 +65,24 @@ impl Command for ToTsv { ) -> Result { let head = call.head; let noheaders = call.has_flag(engine_state, stack, "noheaders")?; - let config = engine_state.get_config(); - to_tsv(input, noheaders, head, config) + let columns: Option> = call.get_flag(engine_state, stack, "columns")?; + let config = engine_state.config.clone(); + to_tsv(input, noheaders, columns, head, config) } } fn to_tsv( input: PipelineData, noheaders: bool, + columns: Option>, head: Span, - config: &Config, + config: Arc, ) -> Result { - to_delimited_data(noheaders, '\t', "TSV", input, head, config) + let sep = Spanned { + item: '\t', + span: head, + }; + to_delimited_data(noheaders, sep, columns, "TSV", input, head, config) } #[cfg(test)] diff --git a/crates/nu-command/src/generators/cal.rs b/crates/nu-command/src/generators/cal.rs index e1ecc771de..018d9370dd 100644 --- a/crates/nu-command/src/generators/cal.rs +++ b/crates/nu-command/src/generators/cal.rs @@ -1,6 +1,7 @@ use chrono::{Datelike, Local, NaiveDate}; use nu_color_config::StyleComputer; use nu_engine::command_prelude::*; +use nu_protocol::ast::{self, Expr, Expression}; use std::collections::VecDeque; @@ -14,6 +15,7 @@ struct Arguments { month_names: bool, full_year: Option>, week_start: Option>, + as_table: bool, } impl Command for Cal { @@ -26,6 +28,7 @@ impl Command for Cal { .switch("year", "Display the year column", Some('y')) .switch("quarter", "Display the quarter column", Some('q')) .switch("month", "Display the month column", Some('m')) + .switch("as-table", "output as a table", Some('t')) .named( "full-year", SyntaxShape::Int, @@ -43,7 +46,10 @@ impl Command for Cal { "Display the month names instead of integers", None, ) - .input_output_types(vec![(Type::Nothing, Type::table())]) + .input_output_types(vec![ + (Type::Nothing, Type::table()), + (Type::Nothing, Type::String), + ]) .allow_variants_without_examples(true) // TODO: supply exhaustive examples .category(Category::Generators) } @@ -75,10 +81,15 @@ impl Command for Cal { result: None, }, Example { - description: "This month's calendar with the week starting on monday", + description: "This month's calendar with the week starting on Monday", example: "cal --week-start mo", result: None, }, + Example { + description: "How many 'Friday the Thirteenths' occurred in 2015?", + example: "cal --as-table --full-year 2015 | where fr == 13 | length", + result: None, + }, ] } } @@ -101,6 +112,7 @@ pub fn cal( quarter: call.has_flag(engine_state, stack, "quarter")?, full_year: call.get_flag(engine_state, stack, "full-year")?, week_start: call.get_flag(engine_state, stack, "week-start")?, + as_table: call.has_flag(engine_state, stack, "as-table")?, }; let style_computer = &StyleComputer::from_config(engine_state, stack); @@ -131,7 +143,32 @@ pub fn cal( style_computer, )?; - Ok(Value::list(calendar_vec_deque.into_iter().collect(), tag).into_pipeline_data()) + let mut table_no_index = ast::Call::new(Span::unknown()); + table_no_index.add_named(( + Spanned { + item: "index".to_string(), + span: Span::unknown(), + }, + None, + Some(Expression::new_unknown( + Expr::Bool(false), + Span::unknown(), + Type::Bool, + )), + )); + + let cal_table_output = + Value::list(calendar_vec_deque.into_iter().collect(), tag).into_pipeline_data(); + if !arguments.as_table { + crate::Table.run( + engine_state, + stack, + &(&table_no_index).into(), + cal_table_output, + ) + } else { + Ok(cal_table_output) + } } fn get_invalid_year_shell_error(head: Span) -> ShellError { diff --git a/crates/nu-command/src/generators/generate.rs b/crates/nu-command/src/generators/generate.rs index 3549667ff0..8d4f48d3fb 100644 --- a/crates/nu-command/src/generators/generate.rs +++ b/crates/nu-command/src/generators/generate.rs @@ -1,4 +1,3 @@ -use itertools::unfold; use nu_engine::{command_prelude::*, ClosureEval}; use nu_protocol::engine::Closure; @@ -12,13 +11,7 @@ impl Command for Generate { fn signature(&self) -> Signature { Signature::build("generate") - .input_output_types(vec![ - (Type::Nothing, Type::List(Box::new(Type::Any))), - ( - Type::List(Box::new(Type::Any)), - Type::List(Box::new(Type::Any)), - ), - ]) + .input_output_types(vec![(Type::Nothing, Type::List(Box::new(Type::Any)))]) .required("initial", SyntaxShape::Any, "Initial value.") .required( "closure", @@ -63,23 +56,10 @@ used as the next argument to the closure, otherwise generation stops. )), }, Example { - example: "generate [0, 1] {|fib| {out: $fib.0, next: [$fib.1, ($fib.0 + $fib.1)]} } | first 10", - description: "Generate a stream of fibonacci numbers", - result: Some(Value::list( - vec![ - Value::test_int(0), - Value::test_int(1), - Value::test_int(1), - Value::test_int(2), - Value::test_int(3), - Value::test_int(5), - Value::test_int(8), - Value::test_int(13), - Value::test_int(21), - Value::test_int(34), - ], - Span::test_data(), - )), + example: + "generate [0, 1] {|fib| {out: $fib.0, next: [$fib.1, ($fib.0 + $fib.1)]} }", + description: "Generate a continuous stream of Fibonacci numbers", + result: None, }, ] } @@ -100,7 +80,8 @@ used as the next argument to the closure, otherwise generation stops. // A type of Option is used to represent state. Invocation // will stop on None. Using Option allows functions to output // one final value before stopping. - let iter = unfold(Some(initial), move |state| { + let mut state = Some(initial); + let iter = std::iter::from_fn(move || { let arg = state.take()?; let (output, next_input) = match closure.run_with_value(arg) { @@ -179,13 +160,13 @@ used as the next argument to the closure, otherwise generation stops. // We use `state` to control when to stop, not `output`. By wrapping // it in a `Some`, we allow the generator to output `None` as a valid output // value. - *state = next_input; + state = next_input; Some(output) }); Ok(iter .flatten() - .into_pipeline_data(call.head, engine_state.ctrlc.clone())) + .into_pipeline_data(call.head, engine_state.signals().clone())) } } diff --git a/crates/nu-command/src/generators/seq.rs b/crates/nu-command/src/generators/seq.rs index 359d479ca0..3636d89d4c 100644 --- a/crates/nu-command/src/generators/seq.rs +++ b/crates/nu-command/src/generators/seq.rs @@ -129,7 +129,7 @@ pub fn run_seq( span, }, span, - engine_state.ctrlc.clone(), + engine_state.signals().clone(), ) } else { ListStream::new( @@ -141,7 +141,7 @@ pub fn run_seq( span, }, span, - engine_state.ctrlc.clone(), + engine_state.signals().clone(), ) }; diff --git a/crates/nu-command/src/hash/generic_digest.rs b/crates/nu-command/src/hash/generic_digest.rs index ab15ccae7a..f098f8a0cb 100644 --- a/crates/nu-command/src/hash/generic_digest.rs +++ b/crates/nu-command/src/hash/generic_digest.rs @@ -96,7 +96,7 @@ where } } else { let args = Arguments { binary, cell_paths }; - operate(action::, args, input, head, engine_state.ctrlc.clone()) + operate(action::, args, input, head, engine_state.signals()) } } } diff --git a/crates/nu-command/src/help/help_commands.rs b/crates/nu-command/src/help/help_commands.rs index f2440cc36f..f067a9fdcb 100644 --- a/crates/nu-command/src/help/help_commands.rs +++ b/crates/nu-command/src/help/help_commands.rs @@ -122,7 +122,7 @@ fn build_help_commands(engine_state: &EngineState, span: Span) -> Vec { let usage = sig.usage; let search_terms = sig.search_terms; - let command_type = format!("{:?}", decl.command_type()).to_ascii_lowercase(); + let command_type = decl.command_type().to_string(); // Build table of parameters let param_table = { diff --git a/crates/nu-command/src/help/help_operators.rs b/crates/nu-command/src/help/help_operators.rs index a7beee3dd5..12803fad8e 100644 --- a/crates/nu-command/src/help/help_operators.rs +++ b/crates/nu-command/src/help/help_operators.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use nu_protocol::ast::{Assignment, Bits, Boolean, Comparison, Math, Operator}; #[derive(Clone)] pub struct HelpOperators; @@ -27,282 +28,148 @@ impl Command for HelpOperators { _input: PipelineData, ) -> Result { let head = call.head; - let op_info = generate_operator_info(); - let mut recs = vec![]; - - for op in op_info { - recs.push(Value::record( + let mut operators = [ + Operator::Assignment(Assignment::Assign), + Operator::Assignment(Assignment::PlusAssign), + Operator::Assignment(Assignment::AppendAssign), + Operator::Assignment(Assignment::MinusAssign), + Operator::Assignment(Assignment::MultiplyAssign), + Operator::Assignment(Assignment::DivideAssign), + Operator::Comparison(Comparison::Equal), + Operator::Comparison(Comparison::NotEqual), + Operator::Comparison(Comparison::LessThan), + Operator::Comparison(Comparison::GreaterThan), + Operator::Comparison(Comparison::LessThanOrEqual), + Operator::Comparison(Comparison::GreaterThanOrEqual), + Operator::Comparison(Comparison::RegexMatch), + Operator::Comparison(Comparison::NotRegexMatch), + Operator::Comparison(Comparison::In), + Operator::Comparison(Comparison::NotIn), + Operator::Comparison(Comparison::StartsWith), + Operator::Comparison(Comparison::EndsWith), + Operator::Math(Math::Plus), + Operator::Math(Math::Append), + Operator::Math(Math::Minus), + Operator::Math(Math::Multiply), + Operator::Math(Math::Divide), + Operator::Math(Math::Modulo), + Operator::Math(Math::FloorDivision), + Operator::Math(Math::Pow), + Operator::Bits(Bits::BitOr), + Operator::Bits(Bits::BitXor), + Operator::Bits(Bits::BitAnd), + Operator::Bits(Bits::ShiftLeft), + Operator::Bits(Bits::ShiftRight), + Operator::Boolean(Boolean::And), + Operator::Boolean(Boolean::Or), + Operator::Boolean(Boolean::Xor), + ] + .into_iter() + .map(|op| { + Value::record( record! { - "type" => Value::string(op.op_type, head), - "operator" => Value::string(op.operator, head), - "name" => Value::string(op.name, head), - "description" => Value::string(op.description, head), - "precedence" => Value::int(op.precedence, head), + "type" => Value::string(op_type(&op), head), + "operator" => Value::string(op.to_string(), head), + "name" => Value::string(name(&op), head), + "description" => Value::string(description(&op), head), + "precedence" => Value::int(op.precedence().into(), head), }, head, - )); - } + ) + }) + .collect::>(); - Ok(Value::list(recs, head).into_pipeline_data()) + operators.push(Value::record( + record! { + "type" => Value::string("Boolean", head), + "operator" => Value::string("not", head), + "name" => Value::string("Not", head), + "description" => Value::string("Negates a value or expression.", head), + "precedence" => Value::int(0, head), + }, + head, + )); + + Ok(Value::list(operators, head).into_pipeline_data()) } } -struct OperatorInfo { - op_type: String, - operator: String, - name: String, - description: String, - precedence: i64, +fn op_type(operator: &Operator) -> &'static str { + match operator { + Operator::Comparison(_) => "Comparison", + Operator::Math(_) => "Math", + Operator::Boolean(_) => "Boolean", + Operator::Bits(_) => "Bitwise", + Operator::Assignment(_) => "Assignment", + } } -fn generate_operator_info() -> Vec { - vec![ - OperatorInfo { - op_type: "Assignment".into(), - operator: "=".into(), - name: "Assign".into(), - description: "Assigns a value to a variable.".into(), - precedence: 10, - }, - OperatorInfo { - op_type: "Assignment".into(), - operator: "+=".into(), - name: "PlusAssign".into(), - description: "Adds a value to a variable.".into(), - precedence: 10, - }, - OperatorInfo { - op_type: "Assignment".into(), - operator: "++=".into(), - name: "AppendAssign".into(), - description: "Appends a list or a value to a variable.".into(), - precedence: 10, - }, - OperatorInfo { - op_type: "Assignment".into(), - operator: "-=".into(), - name: "MinusAssign".into(), - description: "Subtracts a value from a variable.".into(), - precedence: 10, - }, - OperatorInfo { - op_type: "Assignment".into(), - operator: "*=".into(), - name: "MultiplyAssign".into(), - description: "Multiplies a variable by a value.".into(), - precedence: 10, - }, - OperatorInfo { - op_type: "Assignment".into(), - operator: "/=".into(), - name: "DivideAssign".into(), - description: "Divides a variable by a value.".into(), - precedence: 10, - }, - OperatorInfo { - op_type: "Comparison".into(), - operator: "==".into(), - name: "Equal".into(), - description: "Checks if two values are equal.".into(), - precedence: 80, - }, - OperatorInfo { - op_type: "Comparison".into(), - operator: "!=".into(), - name: "NotEqual".into(), - description: "Checks if two values are not equal.".into(), - precedence: 80, - }, - OperatorInfo { - op_type: "Comparison".into(), - operator: "<".into(), - name: "LessThan".into(), - description: "Checks if a value is less than another.".into(), - precedence: 80, - }, - OperatorInfo { - op_type: "Comparison".into(), - operator: "<=".into(), - name: "LessThanOrEqual".into(), - description: "Checks if a value is less than or equal to another.".into(), - precedence: 80, - }, - OperatorInfo { - op_type: "Comparison".into(), - operator: ">".into(), - name: "GreaterThan".into(), - description: "Checks if a value is greater than another.".into(), - precedence: 80, - }, - OperatorInfo { - op_type: "Comparison".into(), - operator: ">=".into(), - name: "GreaterThanOrEqual".into(), - description: "Checks if a value is greater than or equal to another.".into(), - precedence: 80, - }, - OperatorInfo { - op_type: "Comparison".into(), - operator: "=~".into(), - name: "RegexMatch".into(), - description: "Checks if a value matches a regular expression.".into(), - precedence: 80, - }, - OperatorInfo { - op_type: "Comparison".into(), - operator: "!~".into(), - name: "NotRegexMatch".into(), - description: "Checks if a value does not match a regular expression.".into(), - precedence: 80, - }, - OperatorInfo { - op_type: "Comparison".into(), - operator: "in".into(), - name: "In".into(), - description: "Checks if a value is in a list or string.".into(), - precedence: 80, - }, - OperatorInfo { - op_type: "Comparison".into(), - operator: "not-in".into(), - name: "NotIn".into(), - description: "Checks if a value is not in a list or string.".into(), - precedence: 80, - }, - OperatorInfo { - op_type: "Comparison".into(), - operator: "starts-with".into(), - name: "StartsWith".into(), - description: "Checks if a string starts with another.".into(), - precedence: 80, - }, - OperatorInfo { - op_type: "Comparison".into(), - operator: "ends-with".into(), - name: "EndsWith".into(), - description: "Checks if a string ends with another.".into(), - precedence: 80, - }, - OperatorInfo { - op_type: "Comparison".into(), - operator: "not".into(), - name: "UnaryNot".into(), - description: "Negates a value or expression.".into(), - precedence: 0, - }, - OperatorInfo { - op_type: "Math".into(), - operator: "+".into(), - name: "Plus".into(), - description: "Adds two values.".into(), - precedence: 90, - }, - OperatorInfo { - op_type: "Math".into(), - operator: "++".into(), - name: "Append".into(), - description: "Appends two lists or a list and a value.".into(), - precedence: 80, - }, - OperatorInfo { - op_type: "Math".into(), - operator: "-".into(), - name: "Minus".into(), - description: "Subtracts two values.".into(), - precedence: 90, - }, - OperatorInfo { - op_type: "Math".into(), - operator: "*".into(), - name: "Multiply".into(), - description: "Multiplies two values.".into(), - precedence: 95, - }, - OperatorInfo { - op_type: "Math".into(), - operator: "/".into(), - name: "Divide".into(), - description: "Divides two values.".into(), - precedence: 95, - }, - OperatorInfo { - op_type: "Math".into(), - operator: "//".into(), - name: "FloorDivision".into(), - description: "Divides two values and floors the result.".into(), - precedence: 95, - }, - OperatorInfo { - op_type: "Math".into(), - operator: "mod".into(), - name: "Modulo".into(), - description: "Divides two values and returns the remainder.".into(), - precedence: 95, - }, - OperatorInfo { - op_type: "Math".into(), - operator: "**".into(), - name: "Pow ".into(), - description: "Raises one value to the power of another.".into(), - precedence: 100, - }, - OperatorInfo { - op_type: "Bitwise".into(), - operator: "bit-or".into(), - name: "BitOr".into(), - description: "Performs a bitwise OR on two values.".into(), - precedence: 60, - }, - OperatorInfo { - op_type: "Bitwise".into(), - operator: "bit-xor".into(), - name: "BitXor".into(), - description: "Performs a bitwise XOR on two values.".into(), - precedence: 70, - }, - OperatorInfo { - op_type: "Bitwise".into(), - operator: "bit-and".into(), - name: "BitAnd".into(), - description: "Performs a bitwise AND on two values.".into(), - precedence: 75, - }, - OperatorInfo { - op_type: "Bitwise".into(), - operator: "bit-shl".into(), - name: "ShiftLeft".into(), - description: "Shifts a value left by another.".into(), - precedence: 85, - }, - OperatorInfo { - op_type: "Bitwise".into(), - operator: "bit-shr".into(), - name: "ShiftRight".into(), - description: "Shifts a value right by another.".into(), - precedence: 85, - }, - OperatorInfo { - op_type: "Boolean".into(), - operator: "and".into(), - name: "And".into(), - description: "Checks if two values are true.".into(), - precedence: 50, - }, - OperatorInfo { - op_type: "Boolean".into(), - operator: "or".into(), - name: "Or".into(), - description: "Checks if either value is true.".into(), - precedence: 40, - }, - OperatorInfo { - op_type: "Boolean".into(), - operator: "xor".into(), - name: "Xor".into(), - description: "Checks if one value is true and the other is false.".into(), - precedence: 45, - }, - ] +fn name(operator: &Operator) -> String { + match operator { + Operator::Comparison(op) => format!("{op:?}"), + Operator::Math(op) => format!("{op:?}"), + Operator::Boolean(op) => format!("{op:?}"), + Operator::Bits(op) => format!("{op:?}"), + Operator::Assignment(op) => format!("{op:?}"), + } +} + +fn description(operator: &Operator) -> &'static str { + // NOTE: if you have to add an operator here, also add it to the operator list above. + match operator { + Operator::Comparison(Comparison::Equal) => "Checks if two values are equal.", + Operator::Comparison(Comparison::NotEqual) => "Checks if two values are not equal.", + Operator::Comparison(Comparison::LessThan) => "Checks if a value is less than another.", + Operator::Comparison(Comparison::GreaterThan) => { + "Checks if a value is greater than another." + } + Operator::Comparison(Comparison::LessThanOrEqual) => { + "Checks if a value is less than or equal to another." + } + Operator::Comparison(Comparison::GreaterThanOrEqual) => { + "Checks if a value is greater than or equal to another." + } + Operator::Comparison(Comparison::RegexMatch) => { + "Checks if a value matches a regular expression." + } + Operator::Comparison(Comparison::NotRegexMatch) => { + "Checks if a value does not match a regular expression." + } + Operator::Comparison(Comparison::In) => { + "Checks if a value is in a list, is part of a string, or is a key in a record." + } + Operator::Comparison(Comparison::NotIn) => { + "Checks if a value is not in a list, is not part of a string, or is not a key in a record." + } + Operator::Comparison(Comparison::StartsWith) => "Checks if a string starts with another.", + Operator::Comparison(Comparison::EndsWith) => "Checks if a string ends with another.", + Operator::Math(Math::Plus) => "Adds two values.", + Operator::Math(Math::Append) => { + "Appends two lists, a list and a value, two strings, or two binary values." + } + Operator::Math(Math::Minus) => "Subtracts two values.", + Operator::Math(Math::Multiply) => "Multiplies two values.", + Operator::Math(Math::Divide) => "Divides two values.", + Operator::Math(Math::Modulo) => "Divides two values and returns the remainder.", + Operator::Math(Math::FloorDivision) => "Divides two values and floors the result.", + Operator::Math(Math::Pow) => "Raises one value to the power of another.", + Operator::Boolean(Boolean::And) => "Checks if both values are true.", + Operator::Boolean(Boolean::Or) => "Checks if either value is true.", + Operator::Boolean(Boolean::Xor) => "Checks if one value is true and the other is false.", + Operator::Bits(Bits::BitOr) => "Performs a bitwise OR on two values.", + Operator::Bits(Bits::BitXor) => "Performs a bitwise XOR on two values.", + Operator::Bits(Bits::BitAnd) => "Performs a bitwise AND on two values.", + Operator::Bits(Bits::ShiftLeft) => "Bitwise shifts a value left by another.", + Operator::Bits(Bits::ShiftRight) => "Bitwise shifts a value right by another.", + Operator::Assignment(Assignment::Assign) => "Assigns a value to a variable.", + Operator::Assignment(Assignment::PlusAssign) => "Adds a value to a variable.", + Operator::Assignment(Assignment::AppendAssign) => { + "Appends a list, a value, a string, or a binary value to a variable." + } + Operator::Assignment(Assignment::MinusAssign) => "Subtracts a value from a variable.", + Operator::Assignment(Assignment::MultiplyAssign) => "Multiplies a variable by a value.", + Operator::Assignment(Assignment::DivideAssign) => "Divides a variable by a value.", + } } #[cfg(test)] diff --git a/crates/nu-command/src/math/abs.rs b/crates/nu-command/src/math/abs.rs index 8f653142fa..b16097ba8d 100644 --- a/crates/nu-command/src/math/abs.rs +++ b/crates/nu-command/src/math/abs.rs @@ -42,10 +42,7 @@ impl Command for SubCommand { input: PipelineData, ) -> Result { let head = call.head; - input.map( - move |value| abs_helper(value, head), - engine_state.ctrlc.clone(), - ) + input.map(move |value| abs_helper(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/math/ceil.rs b/crates/nu-command/src/math/ceil.rs index 0b0f6d1696..0886ff0bc5 100644 --- a/crates/nu-command/src/math/ceil.rs +++ b/crates/nu-command/src/math/ceil.rs @@ -41,10 +41,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map( - move |value| operate(value, head), - engine_state.ctrlc.clone(), - ) + input.map(move |value| operate(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/math/floor.rs b/crates/nu-command/src/math/floor.rs index e85e3ca674..c2b34c2cf9 100644 --- a/crates/nu-command/src/math/floor.rs +++ b/crates/nu-command/src/math/floor.rs @@ -41,10 +41,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map( - move |value| operate(value, head), - engine_state.ctrlc.clone(), - ) + input.map(move |value| operate(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/math/log.rs b/crates/nu-command/src/math/log.rs index 90fad17daf..7a7e4bd27b 100644 --- a/crates/nu-command/src/math/log.rs +++ b/crates/nu-command/src/math/log.rs @@ -59,7 +59,7 @@ impl Command for SubCommand { let base = base.item; input.map( move |value| operate(value, head, base), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-command/src/math/round.rs b/crates/nu-command/src/math/round.rs index 7693c9a316..eafe669cf1 100644 --- a/crates/nu-command/src/math/round.rs +++ b/crates/nu-command/src/math/round.rs @@ -50,7 +50,7 @@ impl Command for SubCommand { } input.map( move |value| operate(value, head, precision_param), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-command/src/math/sqrt.rs b/crates/nu-command/src/math/sqrt.rs index c9c9765912..a3ab5fbbd8 100644 --- a/crates/nu-command/src/math/sqrt.rs +++ b/crates/nu-command/src/math/sqrt.rs @@ -41,10 +41,7 @@ impl Command for SubCommand { if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } - input.map( - move |value| operate(value, head), - engine_state.ctrlc.clone(), - ) + input.map(move |value| operate(value, head), engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/math/utils.rs b/crates/nu-command/src/math/utils.rs index 9d2c15e15f..62f96ea073 100644 --- a/crates/nu-command/src/math/utils.rs +++ b/crates/nu-command/src/math/utils.rs @@ -1,6 +1,6 @@ use core::slice; use indexmap::IndexMap; -use nu_protocol::{ast::Call, IntoPipelineData, PipelineData, ShellError, Span, Value}; +use nu_protocol::{engine::Call, IntoPipelineData, PipelineData, ShellError, Signals, Span, Value}; pub fn run_with_function( call: &Call, @@ -92,7 +92,7 @@ pub fn calculate( } PipelineData::Value(Value::Range { val, .. }, ..) => { let new_vals: Result, ShellError> = val - .into_range_iter(span, None) + .into_range_iter(span, Signals::empty()) .map(|val| mf(&[val], span, name)) .collect(); diff --git a/crates/nu-command/src/network/http/client.rs b/crates/nu-command/src/network/http/client.rs index 54f7749627..deeb768eea 100644 --- a/crates/nu-command/src/network/http/client.rs +++ b/crates/nu-command/src/network/http/client.rs @@ -5,16 +5,12 @@ use base64::{ Engine, }; use nu_engine::command_prelude::*; -use nu_protocol::ByteStream; +use nu_protocol::{ByteStream, Signals}; use std::{ collections::HashMap, path::PathBuf, str::FromStr, - sync::{ - atomic::AtomicBool, - mpsc::{self, RecvTimeoutError}, - Arc, - }, + sync::mpsc::{self, RecvTimeoutError}, time::Duration, }; use ureq::{Error, ErrorKind, Request, Response}; @@ -117,10 +113,20 @@ pub fn response_to_buffer( _ => None, }; + // Try to guess whether the response is definitely intended to binary or definitely intended to + // be UTF-8 text. Otherwise specify `None` and just guess. This doesn't have to be thorough. + let content_type_lowercase = response.header("content-type").map(|s| s.to_lowercase()); + let response_type = match content_type_lowercase.as_deref() { + Some("application/octet-stream") => ByteStreamType::Binary, + Some(h) if h.contains("charset=utf-8") => ByteStreamType::String, + _ => ByteStreamType::Unknown, + }; + let reader = response.into_reader(); PipelineData::ByteStream( - ByteStream::read(reader, span, engine_state.ctrlc.clone()).with_known_size(buffer_size), + ByteStream::read(reader, span, engine_state.signals().clone(), response_type) + .with_known_size(buffer_size), None, ) } @@ -170,88 +176,131 @@ impl From for ShellErrorOrRequestError { } } +#[derive(Debug)] +pub enum HttpBody { + Value(Value), + ByteStream(ByteStream), + None, +} + +// remove once all commands have been migrated pub fn send_request( request: Request, - body: Option, + http_body: HttpBody, content_type: Option, - ctrl_c: Option>, + span: Span, + signals: &Signals, ) -> Result { let request_url = request.url().to_string(); - if body.is_none() { - return send_cancellable_request(&request_url, Box::new(|| request.call()), ctrl_c); - } - let body = body.expect("Should never be none."); - let body_type = match content_type { - Some(it) if it == "application/json" => BodyType::Json, - Some(it) if it == "application/x-www-form-urlencoded" => BodyType::Form, - _ => BodyType::Unknown, - }; - match body { - Value::Binary { val, .. } => send_cancellable_request( - &request_url, - Box::new(move || request.send_bytes(&val)), - ctrl_c, - ), - Value::String { .. } if body_type == BodyType::Json => { - let data = value_to_json_value(&body)?; - send_cancellable_request(&request_url, Box::new(|| request.send_json(data)), ctrl_c) + match http_body { + HttpBody::None => { + send_cancellable_request(&request_url, Box::new(|| request.call()), span, signals) } - Value::String { val, .. } => send_cancellable_request( - &request_url, - Box::new(move || request.send_string(&val)), - ctrl_c, - ), - Value::Record { .. } if body_type == BodyType::Json => { - let data = value_to_json_value(&body)?; - send_cancellable_request(&request_url, Box::new(|| request.send_json(data)), ctrl_c) - } - Value::Record { val, .. } if body_type == BodyType::Form => { - let mut data: Vec<(String, String)> = Vec::with_capacity(val.len()); - - for (col, val) in val.into_owned() { - data.push((col, val.coerce_into_string()?)) - } - - let request_fn = move || { - // coerce `data` into a shape that send_form() is happy with - let data = data - .iter() - .map(|(a, b)| (a.as_str(), b.as_str())) - .collect::>(); - request.send_form(&data) + HttpBody::ByteStream(byte_stream) => { + let req = if let Some(content_type) = content_type { + request.set("Content-Type", &content_type) + } else { + request }; - send_cancellable_request(&request_url, Box::new(request_fn), ctrl_c) + + send_cancellable_request_bytes(&request_url, req, byte_stream, span, signals) } - Value::List { vals, .. } if body_type == BodyType::Form => { - if vals.len() % 2 != 0 { - return Err(ShellErrorOrRequestError::ShellError(ShellError::IOError { + HttpBody::Value(body) => { + let (body_type, req) = match content_type { + Some(it) if it == "application/json" => (BodyType::Json, request), + Some(it) if it == "application/x-www-form-urlencoded" => (BodyType::Form, request), + Some(it) => { + let r = request.clone().set("Content-Type", &it); + (BodyType::Unknown, r) + } + _ => (BodyType::Unknown, request), + }; + + match body { + Value::Binary { val, .. } => send_cancellable_request( + &request_url, + Box::new(move || req.send_bytes(&val)), + span, + signals, + ), + Value::String { .. } if body_type == BodyType::Json => { + let data = value_to_json_value(&body)?; + send_cancellable_request( + &request_url, + Box::new(|| req.send_json(data)), + span, + signals, + ) + } + Value::String { val, .. } => send_cancellable_request( + &request_url, + Box::new(move || req.send_string(&val)), + span, + signals, + ), + Value::Record { .. } if body_type == BodyType::Json => { + let data = value_to_json_value(&body)?; + send_cancellable_request( + &request_url, + Box::new(|| req.send_json(data)), + span, + signals, + ) + } + Value::Record { val, .. } if body_type == BodyType::Form => { + let mut data: Vec<(String, String)> = Vec::with_capacity(val.len()); + + for (col, val) in val.into_owned() { + data.push((col, val.coerce_into_string()?)) + } + + let request_fn = move || { + // coerce `data` into a shape that send_form() is happy with + let data = data + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + .collect::>(); + req.send_form(&data) + }; + send_cancellable_request(&request_url, Box::new(request_fn), span, signals) + } + Value::List { vals, .. } if body_type == BodyType::Form => { + if vals.len() % 2 != 0 { + return Err(ShellErrorOrRequestError::ShellError(ShellError::IOError { + msg: "unsupported body input".into(), + })); + } + + let data = vals + .chunks(2) + .map(|it| Ok((it[0].coerce_string()?, it[1].coerce_string()?))) + .collect::, ShellErrorOrRequestError>>()?; + + let request_fn = move || { + // coerce `data` into a shape that send_form() is happy with + let data = data + .iter() + .map(|(a, b)| (a.as_str(), b.as_str())) + .collect::>(); + req.send_form(&data) + }; + send_cancellable_request(&request_url, Box::new(request_fn), span, signals) + } + Value::List { .. } if body_type == BodyType::Json => { + let data = value_to_json_value(&body)?; + send_cancellable_request( + &request_url, + Box::new(|| req.send_json(data)), + span, + signals, + ) + } + _ => Err(ShellErrorOrRequestError::ShellError(ShellError::IOError { msg: "unsupported body input".into(), - })); + })), } - - let data = vals - .chunks(2) - .map(|it| Ok((it[0].coerce_string()?, it[1].coerce_string()?))) - .collect::, ShellErrorOrRequestError>>()?; - - let request_fn = move || { - // coerce `data` into a shape that send_form() is happy with - let data = data - .iter() - .map(|(a, b)| (a.as_str(), b.as_str())) - .collect::>(); - request.send_form(&data) - }; - send_cancellable_request(&request_url, Box::new(request_fn), ctrl_c) } - Value::List { .. } if body_type == BodyType::Json => { - let data = value_to_json_value(&body)?; - send_cancellable_request(&request_url, Box::new(|| request.send_json(data)), ctrl_c) - } - _ => Err(ShellErrorOrRequestError::ShellError(ShellError::IOError { - msg: "unsupported body input".into(), - })), } } @@ -260,7 +309,8 @@ pub fn send_request( fn send_cancellable_request( request_url: &str, request_fn: Box Result + Sync + Send>, - ctrl_c: Option>, + span: Span, + signals: &Signals, ) -> Result { let (tx, rx) = mpsc::channel::>(); @@ -275,12 +325,7 @@ fn send_cancellable_request( // ...and poll the channel for responses loop { - if nu_utils::ctrl_c::was_pressed(&ctrl_c) { - // Return early and give up on the background thread. The connection will either time out or be disconnected - return Err(ShellErrorOrRequestError::ShellError( - ShellError::InterruptedByUser { span: None }, - )); - } + signals.check(span)?; // 100ms wait time chosen arbitrarily match rx.recv_timeout(Duration::from_millis(100)) { @@ -295,6 +340,57 @@ fn send_cancellable_request( } } +// Helper method used to make blocking HTTP request calls cancellable with ctrl+c +// ureq functions can block for a long time (default 30s?) while attempting to make an HTTP connection +fn send_cancellable_request_bytes( + request_url: &str, + request: Request, + byte_stream: ByteStream, + span: Span, + signals: &Signals, +) -> Result { + let (tx, rx) = mpsc::channel::>(); + let request_url_string = request_url.to_string(); + + // Make the blocking request on a background thread... + std::thread::Builder::new() + .name("HTTP requester".to_string()) + .spawn(move || { + let ret = byte_stream + .reader() + .ok_or_else(|| { + ShellErrorOrRequestError::ShellError(ShellError::GenericError { + error: "Could not read byte stream".to_string(), + msg: "".into(), + span: None, + help: None, + inner: vec![], + }) + }) + .and_then(|reader| { + request.send(reader).map_err(|e| { + ShellErrorOrRequestError::RequestError(request_url_string, Box::new(e)) + }) + }); + + // may fail if the user has cancelled the operation + let _ = tx.send(ret); + }) + .map_err(ShellError::from)?; + + // ...and poll the channel for responses + loop { + signals.check(span)?; + + // 100ms wait time chosen arbitrarily + match rx.recv_timeout(Duration::from_millis(100)) { + Ok(result) => return result, + Err(RecvTimeoutError::Timeout) => continue, + Err(RecvTimeoutError::Disconnected) => panic!("http response channel disconnected"), + } + } +} + pub fn request_set_timeout( timeout: Option, mut request: Request, diff --git a/crates/nu-command/src/network/http/delete.rs b/crates/nu-command/src/network/http/delete.rs index 77cfa70fb8..f32c9d15bb 100644 --- a/crates/nu-command/src/network/http/delete.rs +++ b/crates/nu-command/src/network/http/delete.rs @@ -1,7 +1,7 @@ use crate::network::http::client::{ check_response_redirection, http_client, http_parse_redirect_mode, http_parse_url, request_add_authorization_header, request_add_custom_headers, request_handle_response, - request_set_timeout, send_request, RequestFlags, + request_set_timeout, send_request, HttpBody, RequestFlags, }; use nu_engine::command_prelude::*; @@ -15,7 +15,7 @@ impl Command for SubCommand { fn signature(&self) -> Signature { Signature::build("http delete") - .input_output_types(vec![(Type::Nothing, Type::Any)]) + .input_output_types(vec![(Type::Any, Type::Any)]) .allow_variants_without_examples(true) .required( "URL", @@ -132,6 +132,11 @@ impl Command for SubCommand { "http delete --content-type application/json --data { field: value } https://www.example.com", result: None, }, + Example { + description: "Perform an HTTP delete with JSON content from a pipeline to example.com", + example: "open foo.json | http delete https://www.example.com", + result: None, + }, ] } } @@ -139,7 +144,7 @@ impl Command for SubCommand { struct Arguments { url: Value, headers: Option, - data: Option, + data: HttpBody, content_type: Option, raw: bool, insecure: bool, @@ -155,13 +160,27 @@ fn run_delete( engine_state: &EngineState, stack: &mut Stack, call: &Call, - _input: PipelineData, + input: PipelineData, ) -> Result { + let (data, maybe_metadata) = call + .get_flag::(engine_state, stack, "data")? + .map(|v| (HttpBody::Value(v), None)) + .unwrap_or_else(|| match input { + PipelineData::Value(v, metadata) => (HttpBody::Value(v), metadata), + PipelineData::ByteStream(byte_stream, metadata) => { + (HttpBody::ByteStream(byte_stream), metadata) + } + _ => (HttpBody::None, None), + }); + let content_type = call + .get_flag(engine_state, stack, "content-type")? + .or_else(|| maybe_metadata.and_then(|m| m.content_type)); + let args = Arguments { url: call.req(engine_state, stack, 0)?, headers: call.get_flag(engine_state, stack, "headers")?, - data: call.get_flag(engine_state, stack, "data")?, - content_type: call.get_flag(engine_state, stack, "content-type")?, + data, + content_type, raw: call.has_flag(engine_state, stack, "raw")?, insecure: call.has_flag(engine_state, stack, "insecure")?, user: call.get_flag(engine_state, stack, "user")?, @@ -184,7 +203,6 @@ fn helper( args: Arguments, ) -> Result { let span = args.url.span(); - let ctrl_c = engine_state.ctrlc.clone(); let (requested_url, _) = http_parse_url(call, span, args.url)?; let redirect_mode = http_parse_redirect_mode(args.redirect)?; @@ -195,7 +213,13 @@ fn helper( request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - let response = send_request(request.clone(), args.data, args.content_type, ctrl_c); + let response = send_request( + request.clone(), + args.data, + args.content_type, + call.head, + engine_state.signals(), + ); let request_flags = RequestFlags { raw: args.raw, diff --git a/crates/nu-command/src/network/http/get.rs b/crates/nu-command/src/network/http/get.rs index e86c44e0a1..3b702404f4 100644 --- a/crates/nu-command/src/network/http/get.rs +++ b/crates/nu-command/src/network/http/get.rs @@ -5,6 +5,8 @@ use crate::network::http::client::{ }; use nu_engine::command_prelude::*; +use super::client::HttpBody; + #[derive(Clone)] pub struct SubCommand; @@ -169,7 +171,6 @@ fn helper( args: Arguments, ) -> Result { let span = args.url.span(); - let ctrl_c = engine_state.ctrlc.clone(); let (requested_url, _) = http_parse_url(call, span, args.url)?; let redirect_mode = http_parse_redirect_mode(args.redirect)?; @@ -180,7 +181,13 @@ fn helper( request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - let response = send_request(request.clone(), None, None, ctrl_c); + let response = send_request( + request.clone(), + HttpBody::None, + None, + call.head, + engine_state.signals(), + ); let request_flags = RequestFlags { raw: args.raw, diff --git a/crates/nu-command/src/network/http/head.rs b/crates/nu-command/src/network/http/head.rs index 875cdc8d8e..797fa138e3 100644 --- a/crates/nu-command/src/network/http/head.rs +++ b/crates/nu-command/src/network/http/head.rs @@ -1,11 +1,11 @@ +use super::client::HttpBody; use crate::network::http::client::{ check_response_redirection, http_client, http_parse_redirect_mode, http_parse_url, request_add_authorization_header, request_add_custom_headers, request_handle_response_headers, request_set_timeout, send_request, }; use nu_engine::command_prelude::*; - -use std::sync::{atomic::AtomicBool, Arc}; +use nu_protocol::Signals; #[derive(Clone)] pub struct SubCommand; @@ -131,9 +131,8 @@ fn run_head( timeout: call.get_flag(engine_state, stack, "max-time")?, redirect: call.get_flag(engine_state, stack, "redirect-mode")?, }; - let ctrl_c = engine_state.ctrlc.clone(); - helper(engine_state, stack, call, args, ctrl_c) + helper(engine_state, stack, call, args, engine_state.signals()) } // Helper function that actually goes to retrieve the resource from the url given @@ -143,7 +142,7 @@ fn helper( stack: &mut Stack, call: &Call, args: Arguments, - ctrlc: Option>, + signals: &Signals, ) -> Result { let span = args.url.span(); let (requested_url, _) = http_parse_url(call, span, args.url)?; @@ -156,7 +155,7 @@ fn helper( request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - let response = send_request(request, None, None, ctrlc); + let response = send_request(request, HttpBody::None, None, call.head, signals); check_response_redirection(redirect_mode, span, &response)?; request_handle_response_headers(span, response) } diff --git a/crates/nu-command/src/network/http/options.rs b/crates/nu-command/src/network/http/options.rs index fe4b7dcb1a..cd9a1f79c1 100644 --- a/crates/nu-command/src/network/http/options.rs +++ b/crates/nu-command/src/network/http/options.rs @@ -4,6 +4,8 @@ use crate::network::http::client::{ }; use nu_engine::command_prelude::*; +use super::client::HttpBody; + #[derive(Clone)] pub struct SubCommand; @@ -149,7 +151,6 @@ fn helper( args: Arguments, ) -> Result { let span = args.url.span(); - let ctrl_c = engine_state.ctrlc.clone(); let (requested_url, _) = http_parse_url(call, span, args.url)?; let client = http_client(args.insecure, RedirectMode::Follow, engine_state, stack)?; @@ -159,7 +160,13 @@ fn helper( request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - let response = send_request(request.clone(), None, None, ctrl_c); + let response = send_request( + request.clone(), + HttpBody::None, + None, + call.head, + engine_state.signals(), + ); // http options' response always showed in header, so we set full to true. // And `raw` is useless too because options method doesn't return body, here we set to true diff --git a/crates/nu-command/src/network/http/patch.rs b/crates/nu-command/src/network/http/patch.rs index ca302702ef..7f49781284 100644 --- a/crates/nu-command/src/network/http/patch.rs +++ b/crates/nu-command/src/network/http/patch.rs @@ -1,7 +1,7 @@ use crate::network::http::client::{ check_response_redirection, http_client, http_parse_redirect_mode, http_parse_url, request_add_authorization_header, request_add_custom_headers, request_handle_response, - request_set_timeout, send_request, RequestFlags, + request_set_timeout, send_request, HttpBody, RequestFlags, }; use nu_engine::command_prelude::*; @@ -15,10 +15,10 @@ impl Command for SubCommand { fn signature(&self) -> Signature { Signature::build("http patch") - .input_output_types(vec![(Type::Nothing, Type::Any)]) + .input_output_types(vec![(Type::Any, Type::Any)]) .allow_variants_without_examples(true) .required("URL", SyntaxShape::String, "The URL to post to.") - .required("data", SyntaxShape::Any, "The contents of the post body.") + .optional("data", SyntaxShape::Any, "The contents of the post body.") .named( "user", SyntaxShape::Any, @@ -124,6 +124,11 @@ impl Command for SubCommand { example: "http patch --content-type application/json https://www.example.com { field: value }", result: None, }, + Example { + description: "Patch JSON content from a pipeline to example.com", + example: "open foo.json | http patch https://www.example.com", + result: None, + }, ] } } @@ -131,7 +136,7 @@ impl Command for SubCommand { struct Arguments { url: Value, headers: Option, - data: Value, + data: HttpBody, content_type: Option, raw: bool, insecure: bool, @@ -147,13 +152,37 @@ fn run_patch( engine_state: &EngineState, stack: &mut Stack, call: &Call, - _input: PipelineData, + input: PipelineData, ) -> Result { + let (data, maybe_metadata) = call + .opt::(engine_state, stack, 1)? + .map(|v| (HttpBody::Value(v), None)) + .unwrap_or_else(|| match input { + PipelineData::Value(v, metadata) => (HttpBody::Value(v), metadata), + PipelineData::ByteStream(byte_stream, metadata) => { + (HttpBody::ByteStream(byte_stream), metadata) + } + _ => (HttpBody::None, None), + }); + let content_type = call + .get_flag(engine_state, stack, "content-type")? + .or_else(|| maybe_metadata.and_then(|m| m.content_type)); + + if let HttpBody::None = data { + return Err(ShellError::GenericError { + error: "Data must be provided either through pipeline or positional argument".into(), + msg: "".into(), + span: Some(call.head), + help: None, + inner: vec![], + }); + } + let args = Arguments { url: call.req(engine_state, stack, 0)?, headers: call.get_flag(engine_state, stack, "headers")?, - data: call.req(engine_state, stack, 1)?, - content_type: call.get_flag(engine_state, stack, "content-type")?, + data, + content_type, raw: call.has_flag(engine_state, stack, "raw")?, insecure: call.has_flag(engine_state, stack, "insecure")?, user: call.get_flag(engine_state, stack, "user")?, @@ -176,7 +205,6 @@ fn helper( args: Arguments, ) -> Result { let span = args.url.span(); - let ctrl_c = engine_state.ctrlc.clone(); let (requested_url, _) = http_parse_url(call, span, args.url)?; let redirect_mode = http_parse_redirect_mode(args.redirect)?; @@ -187,7 +215,13 @@ fn helper( request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - let response = send_request(request.clone(), Some(args.data), args.content_type, ctrl_c); + let response = send_request( + request.clone(), + args.data, + args.content_type, + call.head, + engine_state.signals(), + ); let request_flags = RequestFlags { raw: args.raw, diff --git a/crates/nu-command/src/network/http/post.rs b/crates/nu-command/src/network/http/post.rs index 1f0d7f81b0..ea4ec093f3 100644 --- a/crates/nu-command/src/network/http/post.rs +++ b/crates/nu-command/src/network/http/post.rs @@ -1,7 +1,7 @@ use crate::network::http::client::{ check_response_redirection, http_client, http_parse_redirect_mode, http_parse_url, request_add_authorization_header, request_add_custom_headers, request_handle_response, - request_set_timeout, send_request, RequestFlags, + request_set_timeout, send_request, HttpBody, RequestFlags, }; use nu_engine::command_prelude::*; @@ -15,10 +15,10 @@ impl Command for SubCommand { fn signature(&self) -> Signature { Signature::build("http post") - .input_output_types(vec![(Type::Nothing, Type::Any)]) + .input_output_types(vec![(Type::Any, Type::Any)]) .allow_variants_without_examples(true) .required("URL", SyntaxShape::String, "The URL to post to.") - .required("data", SyntaxShape::Any, "The contents of the post body.") + .optional("data", SyntaxShape::Any, "The contents of the post body. Required unless part of a pipeline.") .named( "user", SyntaxShape::Any, @@ -122,6 +122,11 @@ impl Command for SubCommand { example: "http post --content-type application/json https://www.example.com { field: value }", result: None, }, + Example { + description: "Post JSON content from a pipeline to example.com", + example: "open foo.json | http post https://www.example.com", + result: None, + }, ] } } @@ -129,7 +134,7 @@ impl Command for SubCommand { struct Arguments { url: Value, headers: Option, - data: Value, + data: HttpBody, content_type: Option, raw: bool, insecure: bool, @@ -145,13 +150,37 @@ fn run_post( engine_state: &EngineState, stack: &mut Stack, call: &Call, - _input: PipelineData, + input: PipelineData, ) -> Result { + let (data, maybe_metadata) = call + .opt::(engine_state, stack, 1)? + .map(|v| (HttpBody::Value(v), None)) + .unwrap_or_else(|| match input { + PipelineData::Value(v, metadata) => (HttpBody::Value(v), metadata), + PipelineData::ByteStream(byte_stream, metadata) => { + (HttpBody::ByteStream(byte_stream), metadata) + } + _ => (HttpBody::None, None), + }); + let content_type = call + .get_flag(engine_state, stack, "content-type")? + .or_else(|| maybe_metadata.and_then(|m| m.content_type)); + + if let HttpBody::None = data { + return Err(ShellError::GenericError { + error: "Data must be provided either through pipeline or positional argument".into(), + msg: "".into(), + span: Some(call.head), + help: None, + inner: vec![], + }); + } + let args = Arguments { url: call.req(engine_state, stack, 0)?, headers: call.get_flag(engine_state, stack, "headers")?, - data: call.req(engine_state, stack, 1)?, - content_type: call.get_flag(engine_state, stack, "content-type")?, + data, + content_type, raw: call.has_flag(engine_state, stack, "raw")?, insecure: call.has_flag(engine_state, stack, "insecure")?, user: call.get_flag(engine_state, stack, "user")?, @@ -174,7 +203,6 @@ fn helper( args: Arguments, ) -> Result { let span = args.url.span(); - let ctrl_c = engine_state.ctrlc.clone(); let (requested_url, _) = http_parse_url(call, span, args.url)?; let redirect_mode = http_parse_redirect_mode(args.redirect)?; @@ -185,7 +213,13 @@ fn helper( request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - let response = send_request(request.clone(), Some(args.data), args.content_type, ctrl_c); + let response = send_request( + request.clone(), + args.data, + args.content_type, + call.head, + engine_state.signals(), + ); let request_flags = RequestFlags { raw: args.raw, diff --git a/crates/nu-command/src/network/http/put.rs b/crates/nu-command/src/network/http/put.rs index 8abedd2b4c..e2118ea359 100644 --- a/crates/nu-command/src/network/http/put.rs +++ b/crates/nu-command/src/network/http/put.rs @@ -1,7 +1,7 @@ use crate::network::http::client::{ check_response_redirection, http_client, http_parse_redirect_mode, http_parse_url, request_add_authorization_header, request_add_custom_headers, request_handle_response, - request_set_timeout, send_request, RequestFlags, + request_set_timeout, send_request, HttpBody, RequestFlags, }; use nu_engine::command_prelude::*; @@ -15,10 +15,10 @@ impl Command for SubCommand { fn signature(&self) -> Signature { Signature::build("http put") - .input_output_types(vec![(Type::Nothing, Type::Any)]) + .input_output_types(vec![(Type::Any, Type::Any)]) .allow_variants_without_examples(true) .required("URL", SyntaxShape::String, "The URL to post to.") - .required("data", SyntaxShape::Any, "The contents of the post body.") + .optional("data", SyntaxShape::Any, "The contents of the post body. Required unless part of a pipeline.") .named( "user", SyntaxShape::Any, @@ -122,6 +122,11 @@ impl Command for SubCommand { example: "http put --content-type application/json https://www.example.com { field: value }", result: None, }, + Example { + description: "Put JSON content from a pipeline to example.com", + example: "open foo.json | http put https://www.example.com", + result: None, + }, ] } } @@ -129,7 +134,7 @@ impl Command for SubCommand { struct Arguments { url: Value, headers: Option, - data: Value, + data: HttpBody, content_type: Option, raw: bool, insecure: bool, @@ -145,13 +150,38 @@ fn run_put( engine_state: &EngineState, stack: &mut Stack, call: &Call, - _input: PipelineData, + input: PipelineData, ) -> Result { + let (data, maybe_metadata) = call + .opt::(engine_state, stack, 1)? + .map(|v| (HttpBody::Value(v), None)) + .unwrap_or_else(|| match input { + PipelineData::Value(v, metadata) => (HttpBody::Value(v), metadata), + PipelineData::ByteStream(byte_stream, metadata) => { + (HttpBody::ByteStream(byte_stream), metadata) + } + _ => (HttpBody::None, None), + }); + + if let HttpBody::None = data { + return Err(ShellError::GenericError { + error: "Data must be provided either through pipeline or positional argument".into(), + msg: "".into(), + span: Some(call.head), + help: None, + inner: vec![], + }); + } + + let content_type = call + .get_flag(engine_state, stack, "content-type")? + .or_else(|| maybe_metadata.and_then(|m| m.content_type)); + let args = Arguments { url: call.req(engine_state, stack, 0)?, headers: call.get_flag(engine_state, stack, "headers")?, - data: call.req(engine_state, stack, 1)?, - content_type: call.get_flag(engine_state, stack, "content-type")?, + data, + content_type, raw: call.has_flag(engine_state, stack, "raw")?, insecure: call.has_flag(engine_state, stack, "insecure")?, user: call.get_flag(engine_state, stack, "user")?, @@ -174,7 +204,6 @@ fn helper( args: Arguments, ) -> Result { let span = args.url.span(); - let ctrl_c = engine_state.ctrlc.clone(); let (requested_url, _) = http_parse_url(call, span, args.url)?; let redirect_mode = http_parse_redirect_mode(args.redirect)?; @@ -185,7 +214,13 @@ fn helper( request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; - let response = send_request(request.clone(), Some(args.data), args.content_type, ctrl_c); + let response = send_request( + request.clone(), + args.data, + args.content_type, + call.head, + engine_state.signals(), + ); let request_flags = RequestFlags { raw: args.raw, diff --git a/crates/nu-command/src/network/url/decode.rs b/crates/nu-command/src/network/url/decode.rs index 8789eb13ca..b98e50a56e 100644 --- a/crates/nu-command/src/network/url/decode.rs +++ b/crates/nu-command/src/network/url/decode.rs @@ -48,7 +48,7 @@ impl Command for SubCommand { ) -> Result { let cell_paths: Vec = call.rest(engine_state, stack, 0)?; let args = CellPathOnlyArgs::from(cell_paths); - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/network/url/encode.rs b/crates/nu-command/src/network/url/encode.rs index 845487963b..96e0289903 100644 --- a/crates/nu-command/src/network/url/encode.rs +++ b/crates/nu-command/src/network/url/encode.rs @@ -50,15 +50,9 @@ impl Command for SubCommand { let cell_paths: Vec = call.rest(engine_state, stack, 0)?; let args = CellPathOnlyArgs::from(cell_paths); if call.has_flag(engine_state, stack, "all")? { - operate( - action_all, - args, - input, - call.head, - engine_state.ctrlc.clone(), - ) + operate(action_all, args, input, call.head, engine_state.signals()) } else { - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) } } diff --git a/crates/nu-command/src/path/basename.rs b/crates/nu-command/src/path/basename.rs index 04f7cf380e..8b7082048e 100644 --- a/crates/nu-command/src/path/basename.rs +++ b/crates/nu-command/src/path/basename.rs @@ -61,7 +61,7 @@ impl Command for SubCommand { } input.map( move |value| super::operate(&get_basename, &args, value, head), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } @@ -82,7 +82,7 @@ impl Command for SubCommand { } input.map( move |value| super::operate(&get_basename, &args, value, head), - working_set.permanent().ctrlc.clone(), + working_set.permanent().signals(), ) } diff --git a/crates/nu-command/src/path/dirname.rs b/crates/nu-command/src/path/dirname.rs index 2eb864215e..218091ee34 100644 --- a/crates/nu-command/src/path/dirname.rs +++ b/crates/nu-command/src/path/dirname.rs @@ -69,7 +69,7 @@ impl Command for SubCommand { } input.map( move |value| super::operate(&get_dirname, &args, value, head), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } @@ -91,7 +91,7 @@ impl Command for SubCommand { } input.map( move |value| super::operate(&get_dirname, &args, value, head), - working_set.permanent().ctrlc.clone(), + working_set.permanent().signals(), ) } diff --git a/crates/nu-command/src/path/exists.rs b/crates/nu-command/src/path/exists.rs index a99f04113e..86b00c6024 100644 --- a/crates/nu-command/src/path/exists.rs +++ b/crates/nu-command/src/path/exists.rs @@ -65,7 +65,7 @@ If you need to distinguish dirs and files, please use `path type`."# } input.map( move |value| super::operate(&exists, &args, value, head), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } @@ -87,7 +87,7 @@ If you need to distinguish dirs and files, please use `path type`."# } input.map( move |value| super::operate(&exists, &args, value, head), - working_set.permanent().ctrlc.clone(), + working_set.permanent().signals(), ) } diff --git a/crates/nu-command/src/path/expand.rs b/crates/nu-command/src/path/expand.rs index 18497c6426..ac51978810 100644 --- a/crates/nu-command/src/path/expand.rs +++ b/crates/nu-command/src/path/expand.rs @@ -70,7 +70,7 @@ impl Command for SubCommand { } input.map( move |value| super::operate(&expand, &args, value, head), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } @@ -93,7 +93,7 @@ impl Command for SubCommand { } input.map( move |value| super::operate(&expand, &args, value, head), - working_set.permanent().ctrlc.clone(), + working_set.permanent().signals(), ) } diff --git a/crates/nu-command/src/path/parse.rs b/crates/nu-command/src/path/parse.rs index 039f1012ed..cec2f5c6ac 100644 --- a/crates/nu-command/src/path/parse.rs +++ b/crates/nu-command/src/path/parse.rs @@ -63,7 +63,7 @@ On Windows, an extra 'prefix' column is added."# } input.map( move |value| super::operate(&parse, &args, value, head), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } @@ -84,7 +84,7 @@ On Windows, an extra 'prefix' column is added."# } input.map( move |value| super::operate(&parse, &args, value, head), - working_set.permanent().ctrlc.clone(), + working_set.permanent().signals(), ) } diff --git a/crates/nu-command/src/path/relative_to.rs b/crates/nu-command/src/path/relative_to.rs index 6533f6fa79..35df385d74 100644 --- a/crates/nu-command/src/path/relative_to.rs +++ b/crates/nu-command/src/path/relative_to.rs @@ -67,7 +67,7 @@ path."# } input.map( move |value| super::operate(&relative_to, &args, value, head), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } @@ -88,7 +88,7 @@ path."# } input.map( move |value| super::operate(&relative_to, &args, value, head), - working_set.permanent().ctrlc.clone(), + working_set.permanent().signals(), ) } diff --git a/crates/nu-command/src/path/split.rs b/crates/nu-command/src/path/split.rs index ea8e7e1b1f..80d86fb998 100644 --- a/crates/nu-command/src/path/split.rs +++ b/crates/nu-command/src/path/split.rs @@ -51,7 +51,7 @@ impl Command for SubCommand { } input.map( move |value| super::operate(&split, &args, value, head), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } @@ -70,7 +70,7 @@ impl Command for SubCommand { } input.map( move |value| super::operate(&split, &args, value, head), - working_set.permanent().ctrlc.clone(), + working_set.permanent().signals(), ) } diff --git a/crates/nu-command/src/path/type.rs b/crates/nu-command/src/path/type.rs index 8fbc270445..010fd295ad 100644 --- a/crates/nu-command/src/path/type.rs +++ b/crates/nu-command/src/path/type.rs @@ -1,10 +1,12 @@ use super::PathSubcommandArguments; use nu_engine::command_prelude::*; -use nu_path::expand_tilde; +use nu_path::AbsolutePathBuf; use nu_protocol::engine::StateWorkingSet; -use std::path::Path; +use std::{io, path::Path}; -struct Arguments; +struct Arguments { + pwd: AbsolutePathBuf, +} impl PathSubcommandArguments for Arguments {} @@ -35,7 +37,7 @@ impl Command for SubCommand { fn extra_usage(&self) -> &str { r#"This checks the file system to confirm the path's object type. -If nothing is found, an empty string will be returned."# +If the path does not exist, null will be returned."# } fn is_const(&self) -> bool { @@ -45,20 +47,22 @@ If nothing is found, an empty string will be returned."# fn run( &self, engine_state: &EngineState, - _stack: &mut Stack, + stack: &mut Stack, call: &Call, input: PipelineData, ) -> Result { let head = call.head; - let args = Arguments; + let args = Arguments { + pwd: engine_state.cwd(Some(stack))?, + }; // This doesn't match explicit nulls if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } input.map( - move |value| super::operate(&r#type, &args, value, head), - engine_state.ctrlc.clone(), + move |value| super::operate(&path_type, &args, value, head), + engine_state.signals(), ) } @@ -69,15 +73,17 @@ If nothing is found, an empty string will be returned."# input: PipelineData, ) -> Result { let head = call.head; - let args = Arguments; + let args = Arguments { + pwd: working_set.permanent().cwd(None)?, + }; // This doesn't match explicit nulls if matches!(input, PipelineData::Empty) { return Err(ShellError::PipelineEmpty { dst_span: head }); } input.map( - move |value| super::operate(&r#type, &args, value, head), - working_set.permanent().ctrlc.clone(), + move |value| super::operate(&path_type, &args, value, head), + working_set.permanent().signals(), ) } @@ -97,21 +103,13 @@ If nothing is found, an empty string will be returned."# } } -fn r#type(path: &Path, span: Span, _: &Arguments) -> Value { - let meta = if path.starts_with("~") { - let p = expand_tilde(path); - std::fs::symlink_metadata(p) - } else { - std::fs::symlink_metadata(path) - }; - - Value::string( - match &meta { - Ok(data) => get_file_type(data), - Err(_) => "", - }, - span, - ) +fn path_type(path: &Path, span: Span, args: &Arguments) -> Value { + let path = nu_path::expand_path_with(path, &args.pwd, true); + match path.symlink_metadata() { + Ok(metadata) => Value::string(get_file_type(&metadata), span), + Err(err) if err.kind() == io::ErrorKind::NotFound => Value::nothing(span), + Err(err) => Value::error(err.into_spanned(span).into(), span), + } } fn get_file_type(md: &std::fs::Metadata) -> &str { diff --git a/crates/nu-command/src/platform/ansi/ansi_.rs b/crates/nu-command/src/platform/ansi/ansi_.rs index 9e0b0bba66..23161eb7bf 100644 --- a/crates/nu-command/src/platform/ansi/ansi_.rs +++ b/crates/nu-command/src/platform/ansi/ansi_.rs @@ -1,11 +1,8 @@ use nu_ansi_term::*; use nu_engine::command_prelude::*; -use nu_protocol::engine::StateWorkingSet; +use nu_protocol::{engine::StateWorkingSet, Signals}; use once_cell::sync::Lazy; -use std::{ - collections::HashMap, - sync::{atomic::AtomicBool, Arc}, -}; +use std::collections::HashMap; #[derive(Clone)] pub struct AnsiCommand; @@ -657,10 +654,13 @@ Operating system commands: let escape: bool = call.has_flag(engine_state, stack, "escape")?; let osc: bool = call.has_flag(engine_state, stack, "osc")?; let use_ansi_coloring = engine_state.get_config().use_ansi_coloring; - let ctrlc = engine_state.ctrlc.clone(); if list { - return Ok(generate_ansi_code_list(ctrlc, call.head, use_ansi_coloring)); + return Ok(generate_ansi_code_list( + engine_state.signals().clone(), + call.head, + use_ansi_coloring, + )); } // The code can now be one of the ansi abbreviations like green_bold @@ -676,7 +676,7 @@ Operating system commands: } }; - let output = heavy_lifting(code, escape, osc, call)?; + let output = heavy_lifting(code, escape, osc, stack, call)?; Ok(Value::string(output, call.head).into_pipeline_data()) } @@ -691,10 +691,13 @@ Operating system commands: let escape: bool = call.has_flag_const(working_set, "escape")?; let osc: bool = call.has_flag_const(working_set, "osc")?; let use_ansi_coloring = working_set.get_config().use_ansi_coloring; - let ctrlc = working_set.permanent().ctrlc.clone(); if list { - return Ok(generate_ansi_code_list(ctrlc, call.head, use_ansi_coloring)); + return Ok(generate_ansi_code_list( + working_set.permanent().signals().clone(), + call.head, + use_ansi_coloring, + )); } // The code can now be one of the ansi abbreviations like green_bold @@ -710,26 +713,30 @@ Operating system commands: } }; - let output = heavy_lifting(code, escape, osc, call)?; + let output = heavy_lifting(code, escape, osc, &Stack::new(), call)?; Ok(Value::string(output, call.head).into_pipeline_data()) } } -fn heavy_lifting(code: Value, escape: bool, osc: bool, call: &Call) -> Result { +fn heavy_lifting( + code: Value, + escape: bool, + osc: bool, + stack: &Stack, + call: &Call, +) -> Result { let param_is_string = matches!(code, Value::String { .. }); if escape && osc { return Err(ShellError::IncompatibleParameters { left_message: "escape".into(), left_span: call - .get_named_arg("escape") - .expect("Unexpected missing argument") - .span, + .get_flag_span(stack, "escape") + .expect("Unexpected missing argument"), right_message: "osc".into(), right_span: call - .get_named_arg("osc") - .expect("Unexpected missing argument") - .span, + .get_flag_span(stack, "osc") + .expect("Unexpected missing argument"), }); } let code_string = if param_is_string { @@ -741,10 +748,7 @@ fn heavy_lifting(code: Value, escape: bool, osc: bool, call: &Call) -> Result = code_string.chars().collect(); if code_vec[0] == '\\' { - let span = match call.get_flag_expr("escape") { - Some(expr) => expr.span, - None => call.head, - }; + let span = call.get_flag_span(stack, "escape").unwrap_or(call.head); return Err(ShellError::TypeMismatch { err_message: "no need for escape characters".into(), @@ -827,7 +831,7 @@ pub fn str_to_ansi(s: &str) -> Option { } fn generate_ansi_code_list( - ctrlc: Option>, + signals: Signals, call_span: Span, use_ansi_coloring: bool, ) -> PipelineData { @@ -862,7 +866,7 @@ fn generate_ansi_code_list( Value::record(record, call_span) }) - .into_pipeline_data(call_span, ctrlc) + .into_pipeline_data(call_span, signals.clone()) } fn build_ansi_hashmap(v: &[AnsiCode]) -> HashMap<&str, &str> { diff --git a/crates/nu-command/src/platform/ansi/link.rs b/crates/nu-command/src/platform/ansi/link.rs index 68fc17977b..b45b365d3f 100644 --- a/crates/nu-command/src/platform/ansi/link.rs +++ b/crates/nu-command/src/platform/ansi/link.rs @@ -91,12 +91,12 @@ fn operate( if column_paths.is_empty() { input.map( move |v| process_value(&v, text.as_deref()), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } else { input.map( move |v| process_each_path(v, &column_paths, text.as_deref(), command_span), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } } diff --git a/crates/nu-command/src/platform/ansi/strip.rs b/crates/nu-command/src/platform/ansi/strip.rs index 35d410161c..3d59da6a10 100644 --- a/crates/nu-command/src/platform/ansi/strip.rs +++ b/crates/nu-command/src/platform/ansi/strip.rs @@ -56,7 +56,7 @@ impl Command for SubCommand { cell_paths, config: config.clone(), }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/platform/dir_info.rs b/crates/nu-command/src/platform/dir_info.rs index e4a3039672..10ce4f5420 100644 --- a/crates/nu-command/src/platform/dir_info.rs +++ b/crates/nu-command/src/platform/dir_info.rs @@ -1,10 +1,7 @@ use filesize::file_real_size_fast; use nu_glob::Pattern; -use nu_protocol::{record, ShellError, Span, Value}; -use std::{ - path::PathBuf, - sync::{atomic::AtomicBool, Arc}, -}; +use nu_protocol::{record, ShellError, Signals, Span, Value}; +use std::path::PathBuf; #[derive(Debug, Clone)] pub struct DirBuilder { @@ -82,8 +79,9 @@ impl DirInfo { path: impl Into, params: &DirBuilder, depth: Option, - ctrl_c: Option>, - ) -> Self { + span: Span, + signals: &Signals, + ) -> Result { let path = path.into(); let mut s = Self { @@ -107,14 +105,12 @@ impl DirInfo { match std::fs::read_dir(&s.path) { Ok(d) => { for f in d { - if nu_utils::ctrl_c::was_pressed(&ctrl_c) { - break; - } + signals.check(span)?; match f { Ok(i) => match i.file_type() { Ok(t) if t.is_dir() => { - s = s.add_dir(i.path(), depth, params, ctrl_c.clone()) + s = s.add_dir(i.path(), depth, params, span, signals)? } Ok(_t) => s = s.add_file(i.path(), params), Err(e) => s = s.add_error(e.into()), @@ -125,7 +121,7 @@ impl DirInfo { } Err(e) => s = s.add_error(e.into()), } - s + Ok(s) } fn add_dir( @@ -133,21 +129,22 @@ impl DirInfo { path: impl Into, mut depth: Option, params: &DirBuilder, - ctrl_c: Option>, - ) -> Self { + span: Span, + signals: &Signals, + ) -> Result { if let Some(current) = depth { if let Some(new) = current.checked_sub(1) { depth = Some(new); } else { - return self; + return Ok(self); } } - let d = DirInfo::new(path, params, depth, ctrl_c); + let d = DirInfo::new(path, params, depth, span, signals)?; self.size += d.size; self.blocks += d.blocks; self.dirs.push(d); - self + Ok(self) } fn add_file(mut self, f: impl Into, params: &DirBuilder) -> Self { diff --git a/crates/nu-command/src/platform/is_terminal.rs b/crates/nu-command/src/platform/is_terminal.rs index c67329e839..2195f3ff8a 100644 --- a/crates/nu-command/src/platform/is_terminal.rs +++ b/crates/nu-command/src/platform/is_terminal.rs @@ -58,7 +58,7 @@ impl Command for IsTerminal { _ => { return Err(ShellError::IncompatibleParametersSingle { msg: "Only one stream may be checked".into(), - span: Span::merge_many(call.arguments.iter().map(|arg| arg.span())), + span: call.arguments_span(), }); } }; diff --git a/crates/nu-command/src/platform/kill.rs b/crates/nu-command/src/platform/kill.rs index 2e47ee8c78..1cf6f15f01 100644 --- a/crates/nu-command/src/platform/kill.rs +++ b/crates/nu-command/src/platform/kill.rs @@ -84,27 +84,26 @@ impl Command for Kill { { return Err(ShellError::IncompatibleParameters { left_message: "force".to_string(), - left_span: call - .get_named_arg("force") - .ok_or_else(|| ShellError::GenericError { + left_span: call.get_flag_span(stack, "force").ok_or_else(|| { + ShellError::GenericError { error: "Flag error".into(), msg: "flag force not found".into(), span: Some(call.head), help: None, inner: vec![], - })? - .span, + } + })?, right_message: "signal".to_string(), right_span: Span::merge( - call.get_named_arg("signal") - .ok_or_else(|| ShellError::GenericError { + call.get_flag_span(stack, "signal").ok_or_else(|| { + ShellError::GenericError { error: "Flag error".into(), msg: "flag signal not found".into(), span: Some(call.head), help: None, inner: vec![], - })? - .span, + } + })?, signal_span, ), }); diff --git a/crates/nu-command/src/platform/sleep.rs b/crates/nu-command/src/platform/sleep.rs index ddf429509c..4d5f6ec827 100644 --- a/crates/nu-command/src/platform/sleep.rs +++ b/crates/nu-command/src/platform/sleep.rs @@ -56,12 +56,7 @@ impl Command for Sleep { break; } thread::sleep(CTRL_C_CHECK_INTERVAL.min(time_until_deadline)); - // exit early if Ctrl+C was pressed - if nu_utils::ctrl_c::was_pressed(&engine_state.ctrlc) { - return Err(ShellError::InterruptedByUser { - span: Some(call.head), - }); - } + engine_state.signals().check(call.head)?; } Ok(Value::nothing(call.head).into_pipeline_data()) diff --git a/crates/nu-command/src/random/dice.rs b/crates/nu-command/src/random/dice.rs index 2fb659ad1e..5e3a1b98b6 100644 --- a/crates/nu-command/src/random/dice.rs +++ b/crates/nu-command/src/random/dice.rs @@ -78,7 +78,7 @@ fn dice( Value::int(thread_rng.gen_range(1..sides + 1) as i64, span) }); - Ok(ListStream::new(iter, span, engine_state.ctrlc.clone()).into()) + Ok(ListStream::new(iter, span, engine_state.signals().clone()).into()) } #[cfg(test)] diff --git a/crates/nu-command/src/sort_utils.rs b/crates/nu-command/src/sort_utils.rs index 50d8256353..01bb604eac 100644 --- a/crates/nu-command/src/sort_utils.rs +++ b/crates/nu-command/src/sort_utils.rs @@ -81,7 +81,7 @@ pub fn sort( if let Some(nonexistent) = nonexistent_column(&sort_columns, record.columns()) { return Err(ShellError::CantFindColumn { col_name: nonexistent, - span, + span: Some(span), src_span: val_span, }); } diff --git a/crates/nu-command/src/stor/create.rs b/crates/nu-command/src/stor/create.rs index 630718489b..1e1219889d 100644 --- a/crates/nu-command/src/stor/create.rs +++ b/crates/nu-command/src/stor/create.rs @@ -54,7 +54,10 @@ impl Command for StorCreate { let span = call.head; let table_name: Option = call.get_flag(engine_state, stack, "table-name")?; let columns: Option = call.get_flag(engine_state, stack, "columns")?; - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + engine_state.signals().clone(), + )); process(table_name, span, &db, columns)?; // dbg!(db.clone()); @@ -141,6 +144,8 @@ fn process( #[cfg(test)] mod test { + use nu_protocol::Signals; + use super::*; #[test] @@ -154,7 +159,10 @@ mod test { fn test_process_with_valid_parameters() { let table_name = Some("test_table".to_string()); let span = Span::unknown(); - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); let mut columns = Record::new(); columns.insert( "int_column".to_string(), @@ -170,7 +178,10 @@ mod test { fn test_process_with_missing_table_name() { let table_name = None; let span = Span::unknown(); - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); let mut columns = Record::new(); columns.insert( "int_column".to_string(), @@ -190,7 +201,10 @@ mod test { fn test_process_with_missing_columns() { let table_name = Some("test_table".to_string()); let span = Span::unknown(); - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); let result = process(table_name, span, &db, None); @@ -205,7 +219,10 @@ mod test { fn test_process_with_unsupported_column_data_type() { let table_name = Some("test_table".to_string()); let span = Span::unknown(); - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); let mut columns = Record::new(); let column_datatype = "bogus_data_type".to_string(); columns.insert( diff --git a/crates/nu-command/src/stor/delete.rs b/crates/nu-command/src/stor/delete.rs index 4de4874140..676e0490b0 100644 --- a/crates/nu-command/src/stor/delete.rs +++ b/crates/nu-command/src/stor/delete.rs @@ -1,5 +1,6 @@ use crate::database::{SQLiteDatabase, MEMORY_DB}; use nu_engine::command_prelude::*; +use nu_protocol::Signals; #[derive(Clone)] pub struct StorDelete; @@ -82,7 +83,10 @@ impl Command for StorDelete { } // Open the in-mem database - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); if let Some(new_table_name) = table_name_opt { if let Ok(conn) = db.open_connection() { diff --git a/crates/nu-command/src/stor/export.rs b/crates/nu-command/src/stor/export.rs index 95c5ee9f35..f8255eec49 100644 --- a/crates/nu-command/src/stor/export.rs +++ b/crates/nu-command/src/stor/export.rs @@ -1,5 +1,6 @@ use crate::database::{SQLiteDatabase, MEMORY_DB}; use nu_engine::command_prelude::*; +use nu_protocol::Signals; #[derive(Clone)] pub struct StorExport; @@ -58,7 +59,10 @@ impl Command for StorExport { }; // Open the in-mem database - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); if let Ok(conn) = db.open_connection() { // This uses vacuum. I'm not really sure if this is the best way to do this. diff --git a/crates/nu-command/src/stor/import.rs b/crates/nu-command/src/stor/import.rs index 682694e8bb..20b5b64f2d 100644 --- a/crates/nu-command/src/stor/import.rs +++ b/crates/nu-command/src/stor/import.rs @@ -1,5 +1,6 @@ use crate::database::{SQLiteDatabase, MEMORY_DB}; use nu_engine::command_prelude::*; +use nu_protocol::Signals; #[derive(Clone)] pub struct StorImport; @@ -58,7 +59,10 @@ impl Command for StorImport { }; // Open the in-mem database - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); if let Ok(mut conn) = db.open_connection() { db.restore_database_from_file(&mut conn, file_name) diff --git a/crates/nu-command/src/stor/insert.rs b/crates/nu-command/src/stor/insert.rs index e0c0ad4d28..b6b8d50906 100644 --- a/crates/nu-command/src/stor/insert.rs +++ b/crates/nu-command/src/stor/insert.rs @@ -1,5 +1,6 @@ use crate::database::{values_to_sql, SQLiteDatabase, MEMORY_DB}; use nu_engine::command_prelude::*; +use nu_protocol::Signals; use rusqlite::params_from_iter; #[derive(Clone)] @@ -12,14 +13,17 @@ impl Command for StorInsert { fn signature(&self) -> Signature { Signature::build("stor insert") - .input_output_types(vec![(Type::Nothing, Type::table())]) + .input_output_types(vec![ + (Type::Nothing, Type::table()), + (Type::record(), Type::table()), + ]) .required_named( "table-name", SyntaxShape::String, "name of the table you want to insert into", Some('t'), ) - .required_named( + .named( "data-record", SyntaxShape::Record(vec![]), "a record of column names and column values to insert into the specified table", @@ -39,10 +43,16 @@ impl Command for StorInsert { fn examples(&self) -> Vec { vec![Example { - description: "Insert data the in-memory sqlite database using a data-record of column-name and column-value pairs", - example: "stor insert --table-name nudb --data-record {bool1: true, int1: 5, float1: 1.1, str1: fdncred, datetime1: 2023-04-17}", - result: None, - }] + description: "Insert data the in-memory sqlite database using a data-record of column-name and column-value pairs", + example: "stor insert --table-name nudb --data-record {bool1: true, int1: 5, float1: 1.1, str1: fdncred, datetime1: 2023-04-17}", + result: None, + }, + Example { + description: "Insert data through pipeline input as a record of column-name and column-value pairs", + example: "{bool1: true, int1: 5, float1: 1.1, str1: fdncred, datetime1: 2023-04-17} | stor insert --table-name nudb", + result: None, + }, + ] } fn run( @@ -50,13 +60,19 @@ impl Command for StorInsert { engine_state: &EngineState, stack: &mut Stack, call: &Call, - _input: PipelineData, + input: PipelineData, ) -> Result { let span = call.head; let table_name: Option = call.get_flag(engine_state, stack, "table-name")?; - let columns: Option = call.get_flag(engine_state, stack, "data-record")?; + let data_record: Option = call.get_flag(engine_state, stack, "data-record")?; // let config = engine_state.get_config(); - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); + + // Check if the record is being passed as input or using the data record parameter + let columns = handle(span, data_record, input)?; process(table_name, span, &db, columns)?; @@ -64,11 +80,62 @@ impl Command for StorInsert { } } +fn handle( + span: Span, + data_record: Option, + input: PipelineData, +) -> Result { + match input { + PipelineData::Empty => data_record.ok_or_else(|| ShellError::MissingParameter { + param_name: "requires a record".into(), + span, + }), + PipelineData::Value(value, ..) => { + // Since input is being used, check if the data record parameter is used too + if data_record.is_some() { + return Err(ShellError::GenericError { + error: "Pipeline and Flag both being used".into(), + msg: "Use either pipeline input or '--data-record' parameter".into(), + span: Some(span), + help: None, + inner: vec![], + }); + } + match value { + Value::Record { val, .. } => Ok(val.into_owned()), + val => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "record".into(), + wrong_type: val.get_type().to_string(), + dst_span: Span::unknown(), + src_span: val.span(), + }), + } + } + _ => { + if data_record.is_some() { + return Err(ShellError::GenericError { + error: "Pipeline and Flag both being used".into(), + msg: "Use either pipeline input or '--data-record' parameter".into(), + span: Some(span), + help: None, + inner: vec![], + }); + } + Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "record".into(), + wrong_type: "".into(), + dst_span: span, + src_span: span, + }) + } + } +} + fn process( table_name: Option, span: Span, db: &SQLiteDatabase, - columns: Option, + record: Record, ) -> Result<(), ShellError> { if table_name.is_none() { return Err(ShellError::MissingParameter { @@ -77,54 +144,45 @@ fn process( }); } let new_table_name = table_name.unwrap_or("table".into()); + if let Ok(conn) = db.open_connection() { - match columns { - Some(record) => { - let mut create_stmt = format!("INSERT INTO {} ( ", new_table_name); - let cols = record.columns(); - cols.for_each(|col| { - create_stmt.push_str(&format!("{}, ", col)); - }); - if create_stmt.ends_with(", ") { - create_stmt.pop(); - create_stmt.pop(); - } + let mut create_stmt = format!("INSERT INTO {} ( ", new_table_name); + let cols = record.columns(); + cols.for_each(|col| { + create_stmt.push_str(&format!("{}, ", col)); + }); + if create_stmt.ends_with(", ") { + create_stmt.pop(); + create_stmt.pop(); + } - // Values are set as placeholders. - create_stmt.push_str(") VALUES ( "); - for (index, _) in record.columns().enumerate() { - create_stmt.push_str(&format!("?{}, ", index + 1)); - } + // Values are set as placeholders. + create_stmt.push_str(") VALUES ( "); + for (index, _) in record.columns().enumerate() { + create_stmt.push_str(&format!("?{}, ", index + 1)); + } - if create_stmt.ends_with(", ") { - create_stmt.pop(); - create_stmt.pop(); - } + if create_stmt.ends_with(", ") { + create_stmt.pop(); + create_stmt.pop(); + } - create_stmt.push(')'); + create_stmt.push(')'); - // dbg!(&create_stmt); + // dbg!(&create_stmt); - // Get the params from the passed values - let params = values_to_sql(record.values().cloned())?; + // Get the params from the passed values + let params = values_to_sql(record.values().cloned())?; - conn.execute(&create_stmt, params_from_iter(params)) - .map_err(|err| ShellError::GenericError { - error: "Failed to open SQLite connection in memory from insert".into(), - msg: err.to_string(), - span: Some(Span::test_data()), - help: None, - inner: vec![], - })?; - } - None => { - return Err(ShellError::MissingParameter { - param_name: "requires at least one column".into(), - span, - }); - } - }; - } + conn.execute(&create_stmt, params_from_iter(params)) + .map_err(|err| ShellError::GenericError { + error: "Failed to open SQLite connection in memory from insert".into(), + msg: err.to_string(), + span: Some(Span::test_data()), + help: None, + inner: vec![], + })?; + }; // dbg!(db.clone()); Ok(()) } @@ -144,7 +202,10 @@ mod test { #[test] fn test_process_with_simple_parameters() { - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); let create_stmt = "CREATE TABLE test_process_with_simple_parameters ( int_column INTEGER, real_column REAL, @@ -176,14 +237,17 @@ mod test { ), ); - let result = process(table_name, span, &db, Some(columns)); + let result = process(table_name, span, &db, columns); assert!(result.is_ok()); } #[test] fn test_process_string_with_space() { - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); let create_stmt = "CREATE TABLE test_process_string_with_space ( str_column VARCHAR(255) )"; @@ -201,14 +265,17 @@ mod test { Value::test_string("String With Spaces".to_string()), ); - let result = process(table_name, span, &db, Some(columns)); + let result = process(table_name, span, &db, columns); assert!(result.is_ok()); } #[test] fn test_no_errors_when_string_too_long() { - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); let create_stmt = "CREATE TABLE test_errors_when_string_too_long ( str_column VARCHAR(8) )"; @@ -226,14 +293,17 @@ mod test { Value::test_string("ThisIsALongString".to_string()), ); - let result = process(table_name, span, &db, Some(columns)); + let result = process(table_name, span, &db, columns); // SQLite uses dynamic typing, making any length acceptable for a varchar column assert!(result.is_ok()); } #[test] fn test_no_errors_when_param_is_wrong_type() { - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); let create_stmt = "CREATE TABLE test_errors_when_param_is_wrong_type ( int_column INT )"; @@ -251,14 +321,17 @@ mod test { Value::test_string("ThisIsTheWrongType".to_string()), ); - let result = process(table_name, span, &db, Some(columns)); + let result = process(table_name, span, &db, columns); // SQLite uses dynamic typing, making any type acceptable for a column assert!(result.is_ok()); } #[test] fn test_errors_when_column_doesnt_exist() { - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); let create_stmt = "CREATE TABLE test_errors_when_column_doesnt_exist ( int_column INT )"; @@ -276,14 +349,17 @@ mod test { Value::test_string("ThisIsALongString".to_string()), ); - let result = process(table_name, span, &db, Some(columns)); + let result = process(table_name, span, &db, columns); assert!(result.is_err()); } #[test] fn test_errors_when_table_doesnt_exist() { - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); let table_name = Some("test_errors_when_table_doesnt_exist".to_string()); let span = Span::unknown(); @@ -293,7 +369,7 @@ mod test { Value::test_string("ThisIsALongString".to_string()), ); - let result = process(table_name, span, &db, Some(columns)); + let result = process(table_name, span, &db, columns); assert!(result.is_err()); } diff --git a/crates/nu-command/src/stor/open.rs b/crates/nu-command/src/stor/open.rs index c7f6f9f746..ba0f17c2af 100644 --- a/crates/nu-command/src/stor/open.rs +++ b/crates/nu-command/src/stor/open.rs @@ -1,5 +1,6 @@ use crate::database::{SQLiteDatabase, MEMORY_DB}; use nu_engine::command_prelude::*; +use nu_protocol::Signals; #[derive(Clone)] pub struct StorOpen; @@ -54,7 +55,10 @@ impl Command for StorOpen { // It returns the output of `select * from my_table_name` // Just create an empty database with MEMORY_DB and nothing else - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); // dbg!(db.clone()); Ok(db.into_value(call.head).into_pipeline_data()) diff --git a/crates/nu-command/src/stor/reset.rs b/crates/nu-command/src/stor/reset.rs index d4489fb702..ba9e2a9681 100644 --- a/crates/nu-command/src/stor/reset.rs +++ b/crates/nu-command/src/stor/reset.rs @@ -1,5 +1,6 @@ use crate::database::{SQLiteDatabase, MEMORY_DB}; use nu_engine::command_prelude::*; +use nu_protocol::Signals; #[derive(Clone)] pub struct StorReset; @@ -42,7 +43,10 @@ impl Command for StorReset { let span = call.head; // Open the in-mem database - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); if let Ok(conn) = db.open_connection() { db.drop_all_tables(&conn) diff --git a/crates/nu-command/src/stor/update.rs b/crates/nu-command/src/stor/update.rs index d50614d67f..18cf6f9f0f 100644 --- a/crates/nu-command/src/stor/update.rs +++ b/crates/nu-command/src/stor/update.rs @@ -1,5 +1,6 @@ use crate::database::{SQLiteDatabase, MEMORY_DB}; use nu_engine::command_prelude::*; +use nu_protocol::Signals; #[derive(Clone)] pub struct StorUpdate; @@ -11,14 +12,17 @@ impl Command for StorUpdate { fn signature(&self) -> Signature { Signature::build("stor update") - .input_output_types(vec![(Type::Nothing, Type::table())]) + .input_output_types(vec![ + (Type::Nothing, Type::table()), + (Type::record(), Type::table()), + ]) .required_named( "table-name", SyntaxShape::String, "name of the table you want to insert into", Some('t'), ) - .required_named( + .named( "update-record", SyntaxShape::Record(vec![]), "a record of column names and column values to update in the specified table", @@ -54,6 +58,11 @@ impl Command for StorUpdate { example: "stor update --table-name nudb --update-record {str1: nushell datetime1: 2020-04-17} --where-clause \"bool1 = 1\"", result: None, }, + Example { + description: "Update the in-memory sqlite database through pipeline input", + example: "{str1: nushell datetime1: 2020-04-17} | stor update --table-name nudb", + result: None, + }, ] } @@ -62,91 +71,150 @@ impl Command for StorUpdate { engine_state: &EngineState, stack: &mut Stack, call: &Call, - _input: PipelineData, + input: PipelineData, ) -> Result { let span = call.head; let table_name: Option = call.get_flag(engine_state, stack, "table-name")?; - let columns: Option = call.get_flag(engine_state, stack, "update-record")?; + let update_record: Option = call.get_flag(engine_state, stack, "update-record")?; let where_clause_opt: Option> = call.get_flag(engine_state, stack, "where-clause")?; // Open the in-mem database - let db = Box::new(SQLiteDatabase::new(std::path::Path::new(MEMORY_DB), None)); + let db = Box::new(SQLiteDatabase::new( + std::path::Path::new(MEMORY_DB), + Signals::empty(), + )); - if table_name.is_none() { - return Err(ShellError::MissingParameter { - param_name: "requires at table name".into(), - span, - }); - } - let new_table_name = table_name.unwrap_or("table".into()); - if let Ok(conn) = db.open_connection() { - match columns { - Some(record) => { - let mut update_stmt = format!("UPDATE {} ", new_table_name); + // Check if the record is being passed as input or using the update record parameter + let columns = handle(span, update_record, input)?; - update_stmt.push_str("SET "); - let vals = record.iter(); - vals.for_each(|(key, val)| match val { - Value::Int { val, .. } => { - update_stmt.push_str(&format!("{} = {}, ", key, val)); - } - Value::Float { val, .. } => { - update_stmt.push_str(&format!("{} = {}, ", key, val)); - } - Value::String { val, .. } => { - update_stmt.push_str(&format!("{} = '{}', ", key, val)); - } - Value::Date { val, .. } => { - update_stmt.push_str(&format!("{} = '{}', ", key, val)); - } - Value::Bool { val, .. } => { - update_stmt.push_str(&format!("{} = {}, ", key, val)); - } - _ => { - // return Err(ShellError::UnsupportedInput { - // msg: format!("{} is not a valid datepart, expected one of year, month, day, hour, minute, second, millisecond, microsecond, nanosecond", part.item), - // input: "value originates from here".to_string(), - // msg_span: span, - // input_span: val.span(), - // }); - } - }); - if update_stmt.ends_with(", ") { - update_stmt.pop(); - update_stmt.pop(); - } + process(table_name, span, &db, columns, where_clause_opt)?; - // Yup, this is a bit janky, but I'm not sure a better way to do this without having - // --and and --or flags as well as supporting ==, !=, <>, is null, is not null, etc. - // and other sql syntax. So, for now, just type a sql where clause as a string. - if let Some(where_clause) = where_clause_opt { - update_stmt.push_str(&format!(" WHERE {}", where_clause.item)); - } - // dbg!(&update_stmt); - - conn.execute(&update_stmt, []) - .map_err(|err| ShellError::GenericError { - error: "Failed to open SQLite connection in memory from update".into(), - msg: err.to_string(), - span: Some(Span::test_data()), - help: None, - inner: vec![], - })?; - } - None => { - return Err(ShellError::MissingParameter { - param_name: "requires at least one column".into(), - span: call.head, - }); - } - }; - } - // dbg!(db.clone()); Ok(Value::custom(db, span).into_pipeline_data()) } } +fn handle( + span: Span, + update_record: Option, + input: PipelineData, +) -> Result { + match input { + PipelineData::Empty => update_record.ok_or_else(|| ShellError::MissingParameter { + param_name: "requires a record".into(), + span, + }), + PipelineData::Value(value, ..) => { + // Since input is being used, check if the data record parameter is used too + if update_record.is_some() { + return Err(ShellError::GenericError { + error: "Pipeline and Flag both being used".into(), + msg: "Use either pipeline input or '--update-record' parameter".into(), + span: Some(span), + help: None, + inner: vec![], + }); + } + match value { + Value::Record { val, .. } => Ok(val.into_owned()), + val => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "record".into(), + wrong_type: val.get_type().to_string(), + dst_span: Span::unknown(), + src_span: val.span(), + }), + } + } + _ => { + if update_record.is_some() { + return Err(ShellError::GenericError { + error: "Pipeline and Flag both being used".into(), + msg: "Use either pipeline input or '--update-record' parameter".into(), + span: Some(span), + help: None, + inner: vec![], + }); + } + Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "record".into(), + wrong_type: "".into(), + dst_span: span, + src_span: span, + }) + } + } +} + +fn process( + table_name: Option, + span: Span, + db: &SQLiteDatabase, + record: Record, + where_clause_opt: Option>, +) -> Result<(), ShellError> { + if table_name.is_none() { + return Err(ShellError::MissingParameter { + param_name: "requires at table name".into(), + span, + }); + } + let new_table_name = table_name.unwrap_or("table".into()); + if let Ok(conn) = db.open_connection() { + let mut update_stmt = format!("UPDATE {} ", new_table_name); + + update_stmt.push_str("SET "); + let vals = record.iter(); + vals.for_each(|(key, val)| match val { + Value::Int { val, .. } => { + update_stmt.push_str(&format!("{} = {}, ", key, val)); + } + Value::Float { val, .. } => { + update_stmt.push_str(&format!("{} = {}, ", key, val)); + } + Value::String { val, .. } => { + update_stmt.push_str(&format!("{} = '{}', ", key, val)); + } + Value::Date { val, .. } => { + update_stmt.push_str(&format!("{} = '{}', ", key, val)); + } + Value::Bool { val, .. } => { + update_stmt.push_str(&format!("{} = {}, ", key, val)); + } + _ => { + // return Err(ShellError::UnsupportedInput { + // msg: format!("{} is not a valid datepart, expected one of year, month, day, hour, minute, second, millisecond, microsecond, nanosecond", part.item), + // input: "value originates from here".to_string(), + // msg_span: span, + // input_span: val.span(), + // }); + } + }); + if update_stmt.ends_with(", ") { + update_stmt.pop(); + update_stmt.pop(); + } + + // Yup, this is a bit janky, but I'm not sure a better way to do this without having + // --and and --or flags as well as supporting ==, !=, <>, is null, is not null, etc. + // and other sql syntax. So, for now, just type a sql where clause as a string. + if let Some(where_clause) = where_clause_opt { + update_stmt.push_str(&format!(" WHERE {}", where_clause.item)); + } + // dbg!(&update_stmt); + + conn.execute(&update_stmt, []) + .map_err(|err| ShellError::GenericError { + error: "Failed to open SQLite connection in memory from update".into(), + msg: err.to_string(), + span: Some(Span::test_data()), + help: None, + inner: vec![], + })?; + } + // dbg!(db.clone()); + Ok(()) +} + #[cfg(test)] mod test { use super::*; diff --git a/crates/nu-command/src/strings/char_.rs b/crates/nu-command/src/strings/char_.rs index 7d8d152b56..7737210254 100644 --- a/crates/nu-command/src/strings/char_.rs +++ b/crates/nu-command/src/strings/char_.rs @@ -1,8 +1,8 @@ use indexmap::{indexmap, IndexMap}; use nu_engine::command_prelude::*; -use nu_protocol::engine::StateWorkingSet; + +use nu_protocol::Signals; use once_cell::sync::Lazy; -use std::sync::{atomic::AtomicBool, Arc}; // Character used to separate directories in a Path Environment variable on windows is ";" #[cfg(target_family = "windows")] @@ -19,6 +19,9 @@ static CHAR_MAP: Lazy> = Lazy::new(|| { // These are some regular characters that either can't be used or // it's just easier to use them like this. + "nul" => '\x00'.to_string(), // nul character, 0x00 + "null_byte" => '\x00'.to_string(), // nul character, 0x00 + "zero_byte" => '\x00'.to_string(), // nul character, 0x00 // This are the "normal" characters section "newline" => '\n'.to_string(), "enter" => '\n'.to_string(), @@ -226,11 +229,13 @@ impl Command for Char { let list = call.has_flag_const(working_set, "list")?; let integer = call.has_flag_const(working_set, "integer")?; let unicode = call.has_flag_const(working_set, "unicode")?; - let ctrlc = working_set.permanent().ctrlc.clone(); // handle -l flag if list { - return Ok(generate_character_list(ctrlc, call.head)); + return Ok(generate_character_list( + working_set.permanent().signals().clone(), + call.head, + )); } // handle -i flag @@ -261,11 +266,13 @@ impl Command for Char { let list = call.has_flag(engine_state, stack, "list")?; let integer = call.has_flag(engine_state, stack, "integer")?; let unicode = call.has_flag(engine_state, stack, "unicode")?; - let ctrlc = engine_state.ctrlc.clone(); // handle -l flag if list { - return Ok(generate_character_list(ctrlc, call_span)); + return Ok(generate_character_list( + engine_state.signals().clone(), + call_span, + )); } // handle -i flag @@ -286,7 +293,7 @@ impl Command for Char { } } -fn generate_character_list(ctrlc: Option>, call_span: Span) -> PipelineData { +fn generate_character_list(signals: Signals, call_span: Span) -> PipelineData { CHAR_MAP .iter() .map(move |(name, s)| { @@ -305,7 +312,7 @@ fn generate_character_list(ctrlc: Option>, call_span: Span) -> P Value::record(record, call_span) }) - .into_pipeline_data(call_span, ctrlc) + .into_pipeline_data(call_span, signals) } fn handle_integer_flag( diff --git a/crates/nu-command/src/strings/detect_columns.rs b/crates/nu-command/src/strings/detect_columns.rs index 74bcf07820..2f1f713b33 100644 --- a/crates/nu-command/src/strings/detect_columns.rs +++ b/crates/nu-command/src/strings/detect_columns.rs @@ -45,20 +45,6 @@ impl Command for DetectColumns { vec!["split", "tabular"] } - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - if call.has_flag(engine_state, stack, "guess")? { - guess_width(engine_state, stack, call, input) - } else { - detect_columns(engine_state, stack, call, input) - } - } - fn examples(&self) -> Vec { vec![ Example { @@ -109,33 +95,87 @@ none 8150224 4 8150220 1% /mnt/c' | detect columns --gue }, ] } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let num_rows_to_skip: Option = call.get_flag(engine_state, stack, "skip")?; + let noheader = call.has_flag(engine_state, stack, "no-headers")?; + let range: Option = call.get_flag(engine_state, stack, "combine-columns")?; + + let args = Arguments { + noheader, + num_rows_to_skip, + range, + }; + + if call.has_flag(engine_state, stack, "guess")? { + guess_width(engine_state, call, input, args) + } else { + detect_columns(engine_state, call, input, args) + } + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let num_rows_to_skip: Option = call.get_flag_const(working_set, "skip")?; + let noheader = call.has_flag_const(working_set, "no-headers")?; + let range: Option = call.get_flag_const(working_set, "combine-columns")?; + + let args = Arguments { + noheader, + num_rows_to_skip, + range, + }; + + if call.has_flag_const(working_set, "guess")? { + guess_width(working_set.permanent(), call, input, args) + } else { + detect_columns(working_set.permanent(), call, input, args) + } + } +} + +struct Arguments { + num_rows_to_skip: Option, + noheader: bool, + range: Option, } fn guess_width( engine_state: &EngineState, - stack: &mut Stack, call: &Call, input: PipelineData, + args: Arguments, ) -> Result { use super::guess_width::GuessWidth; let input_span = input.span().unwrap_or(call.head); let mut input = input.collect_string("", engine_state.get_config())?; - let num_rows_to_skip: Option = call.get_flag(engine_state, stack, "skip")?; - if let Some(rows) = num_rows_to_skip { + if let Some(rows) = args.num_rows_to_skip { input = input.lines().skip(rows).map(|x| x.to_string()).join("\n"); } let mut guess_width = GuessWidth::new_reader(Box::new(Cursor::new(input))); - let noheader = call.has_flag(engine_state, stack, "no-headers")?; let result = guess_width.read_all(); if result.is_empty() { return Ok(Value::nothing(input_span).into_pipeline_data()); } - let range: Option = call.get_flag(engine_state, stack, "combine-columns")?; - if !noheader { + if !args.noheader { let columns = result[0].clone(); Ok(result .into_iter() @@ -152,14 +192,14 @@ fn guess_width( let record = Record::from_raw_cols_vals(columns.clone(), values, input_span, input_span); match record { - Ok(r) => match &range { + Ok(r) => match &args.range { Some(range) => merge_record(r, range, input_span), None => Value::record(r, input_span), }, Err(e) => Value::error(e, input_span), } }) - .into_pipeline_data(input_span, engine_state.ctrlc.clone())) + .into_pipeline_data(input_span, engine_state.signals().clone())) } else { let length = result[0].len(); let columns: Vec = (0..length).map(|n| format!("column{n}")).collect(); @@ -177,34 +217,30 @@ fn guess_width( let record = Record::from_raw_cols_vals(columns.clone(), values, input_span, input_span); match record { - Ok(r) => match &range { + Ok(r) => match &args.range { Some(range) => merge_record(r, range, input_span), None => Value::record(r, input_span), }, Err(e) => Value::error(e, input_span), } }) - .into_pipeline_data(input_span, engine_state.ctrlc.clone())) + .into_pipeline_data(input_span, engine_state.signals().clone())) } } fn detect_columns( engine_state: &EngineState, - stack: &mut Stack, call: &Call, input: PipelineData, + args: Arguments, ) -> Result { let name_span = call.head; - let num_rows_to_skip: Option = call.get_flag(engine_state, stack, "skip")?; - let noheader = call.has_flag(engine_state, stack, "no-headers")?; - let range: Option = call.get_flag(engine_state, stack, "combine-columns")?; - let ctrlc = engine_state.ctrlc.clone(); let config = engine_state.get_config(); let input = input.collect_string("", config)?; let input: Vec<_> = input .lines() - .skip(num_rows_to_skip.unwrap_or_default()) + .skip(args.num_rows_to_skip.unwrap_or_default()) .map(|x| x.to_string()) .collect(); @@ -214,13 +250,14 @@ fn detect_columns( if let Some(orig_headers) = headers { let mut headers = find_columns(&orig_headers); - if noheader { + if args.noheader { for header in headers.iter_mut().enumerate() { header.1.item = format!("column{}", header.0); } } - Ok(noheader + Ok(args + .noheader .then_some(orig_headers) .into_iter() .chain(input) @@ -273,12 +310,12 @@ fn detect_columns( } } - match &range { + match &args.range { Some(range) => merge_record(record, range, name_span), None => Value::record(record, name_span), } }) - .into_pipeline_data(call.head, ctrlc)) + .into_pipeline_data(call.head, engine_state.signals().clone())) } else { Ok(PipelineData::empty()) } diff --git a/crates/nu-command/src/strings/encode_decode/base64.rs b/crates/nu-command/src/strings/encode_decode/base64.rs index 1c257e1200..dd9289a141 100644 --- a/crates/nu-command/src/strings/encode_decode/base64.rs +++ b/crates/nu-command/src/strings/encode_decode/base64.rs @@ -7,10 +7,9 @@ use base64::{ Engine, }; use nu_cmd_base::input_handler::{operate as general_operate, CmdArgument}; -use nu_engine::CallExt; use nu_protocol::{ - ast::{Call, CellPath}, - engine::{EngineState, Stack}, + ast::CellPath, + engine::{Call, EngineState}, PipelineData, ShellError, Span, Spanned, Value, }; @@ -42,22 +41,24 @@ impl CmdArgument for Arguments { } } +pub(super) struct Base64CommandArguments { + pub(super) character_set: Option>, + pub(super) action_type: ActionType, + pub(super) binary: bool, +} + pub fn operate( - action_type: ActionType, engine_state: &EngineState, - stack: &mut Stack, call: &Call, input: PipelineData, + cell_paths: Vec, + args: Base64CommandArguments, ) -> Result { let head = call.head; - let character_set: Option> = - call.get_flag(engine_state, stack, "character-set")?; - let binary = call.has_flag(engine_state, stack, "binary")?; - let cell_paths: Vec = call.rest(engine_state, stack, 0)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); // Default the character set to standard if the argument is not specified. - let character_set = match character_set { + let character_set = match args.character_set { Some(inner_tag) => inner_tag, None => Spanned { item: "standard".to_string(), @@ -68,13 +69,13 @@ pub fn operate( let args = Arguments { encoding_config: Base64Config { character_set, - action_type, + action_type: args.action_type, }, - binary, + binary: args.binary, cell_paths, }; - general_operate(action, args, input, call.head, engine_state.ctrlc.clone()) + general_operate(action, args, input, call.head, engine_state.signals()) } fn action( diff --git a/crates/nu-command/src/strings/encode_decode/decode.rs b/crates/nu-command/src/strings/encode_decode/decode.rs index 9b13fad202..20385612f3 100644 --- a/crates/nu-command/src/strings/encode_decode/decode.rs +++ b/crates/nu-command/src/strings/encode_decode/decode.rs @@ -46,6 +46,10 @@ documentation link at https://docs.rs/encoding_rs/latest/encoding_rs/#statics"# ] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -53,49 +57,67 @@ documentation link at https://docs.rs/encoding_rs/latest/encoding_rs/#statics"# call: &Call, input: PipelineData, ) -> Result { - let head = call.head; let encoding: Option> = call.opt(engine_state, stack, 0)?; + run(call, input, encoding) + } - match input { - PipelineData::ByteStream(stream, ..) => { - let span = stream.span(); - let bytes = stream.into_bytes()?; - match encoding { + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding: Option> = call.opt_const(working_set, 0)?; + run(call, input, encoding) + } +} + +fn run( + call: &Call, + input: PipelineData, + encoding: Option>, +) -> Result { + let head = call.head; + + match input { + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + let bytes = stream.into_bytes()?; + match encoding { + Some(encoding_name) => super::encoding::decode(head, encoding_name, &bytes), + None => super::encoding::detect_encoding_name(head, span, &bytes) + .map(|encoding| encoding.decode(&bytes).0.into_owned()) + .map(|s| Value::string(s, head)), + } + .map(|val| val.into_pipeline_data()) + } + PipelineData::Value(v, ..) => { + let input_span = v.span(); + match v { + Value::Binary { val: bytes, .. } => match encoding { Some(encoding_name) => super::encoding::decode(head, encoding_name, &bytes), - None => super::encoding::detect_encoding_name(head, span, &bytes) + None => super::encoding::detect_encoding_name(head, input_span, &bytes) .map(|encoding| encoding.decode(&bytes).0.into_owned()) .map(|s| Value::string(s, head)), } - .map(|val| val.into_pipeline_data()) + .map(|val| val.into_pipeline_data()), + Value::Error { error, .. } => Err(*error), + _ => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "binary".into(), + wrong_type: v.get_type().to_string(), + dst_span: head, + src_span: v.span(), + }), } - PipelineData::Value(v, ..) => { - let input_span = v.span(); - match v { - Value::Binary { val: bytes, .. } => match encoding { - Some(encoding_name) => super::encoding::decode(head, encoding_name, &bytes), - None => super::encoding::detect_encoding_name(head, input_span, &bytes) - .map(|encoding| encoding.decode(&bytes).0.into_owned()) - .map(|s| Value::string(s, head)), - } - .map(|val| val.into_pipeline_data()), - Value::Error { error, .. } => Err(*error), - _ => Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "binary".into(), - wrong_type: v.get_type().to_string(), - dst_span: head, - src_span: v.span(), - }), - } - } - // This should be more precise, but due to difficulties in getting spans - // from PipelineData::ListData, this is as it is. - _ => Err(ShellError::UnsupportedInput { - msg: "non-binary input".into(), - input: "value originates from here".into(), - msg_span: head, - input_span: input.span().unwrap_or(head), - }), } + // This should be more precise, but due to difficulties in getting spans + // from PipelineData::ListData, this is as it is. + _ => Err(ShellError::UnsupportedInput { + msg: "non-binary input".into(), + input: "value originates from here".into(), + msg_span: head, + input_span: input.span().unwrap_or(head), + }), } } diff --git a/crates/nu-command/src/strings/encode_decode/decode_base64.rs b/crates/nu-command/src/strings/encode_decode/decode_base64.rs index 242a99bb88..cda9de4be8 100644 --- a/crates/nu-command/src/strings/encode_decode/decode_base64.rs +++ b/crates/nu-command/src/strings/encode_decode/decode_base64.rs @@ -1,4 +1,4 @@ -use super::base64::{operate, ActionType, CHARACTER_SET_DESC}; +use super::base64::{operate, ActionType, Base64CommandArguments, CHARACTER_SET_DESC}; use nu_engine::command_prelude::*; #[derive(Clone)] @@ -66,6 +66,10 @@ impl Command for DecodeBase64 { ] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -73,7 +77,34 @@ impl Command for DecodeBase64 { call: &Call, input: PipelineData, ) -> Result { - operate(ActionType::Decode, engine_state, stack, call, input) + let character_set: Option> = + call.get_flag(engine_state, stack, "character-set")?; + let binary = call.has_flag(engine_state, stack, "binary")?; + let cell_paths: Vec = call.rest(engine_state, stack, 0)?; + let args = Base64CommandArguments { + action_type: ActionType::Decode, + binary, + character_set, + }; + operate(engine_state, call, input, cell_paths, args) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let character_set: Option> = + call.get_flag_const(working_set, "character-set")?; + let binary = call.has_flag_const(working_set, "binary")?; + let cell_paths: Vec = call.rest_const(working_set, 0)?; + let args = Base64CommandArguments { + action_type: ActionType::Decode, + binary, + character_set, + }; + operate(working_set.permanent(), call, input, cell_paths, args) } } diff --git a/crates/nu-command/src/strings/encode_decode/encode.rs b/crates/nu-command/src/strings/encode_decode/encode.rs index 113c0fe548..b7bebdba80 100644 --- a/crates/nu-command/src/strings/encode_decode/encode.rs +++ b/crates/nu-command/src/strings/encode_decode/encode.rs @@ -69,6 +69,10 @@ documentation link at https://docs.rs/encoding_rs/latest/encoding_rs/#statics"# ] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -76,42 +80,62 @@ documentation link at https://docs.rs/encoding_rs/latest/encoding_rs/#statics"# call: &Call, input: PipelineData, ) -> Result { - let head = call.head; let encoding: Spanned = call.req(engine_state, stack, 0)?; let ignore_errors = call.has_flag(engine_state, stack, "ignore-errors")?; + run(call, input, encoding, ignore_errors) + } - match input { - PipelineData::ByteStream(stream, ..) => { - let span = stream.span(); - let s = stream.into_string()?; - super::encoding::encode(head, encoding, &s, span, ignore_errors) - .map(|val| val.into_pipeline_data()) - } - PipelineData::Value(v, ..) => { - let span = v.span(); - match v { - Value::String { val: s, .. } => { - super::encoding::encode(head, encoding, &s, span, ignore_errors) - .map(|val| val.into_pipeline_data()) - } - Value::Error { error, .. } => Err(*error), - _ => Err(ShellError::OnlySupportsThisInputType { - exp_input_type: "string".into(), - wrong_type: v.get_type().to_string(), - dst_span: head, - src_span: v.span(), - }), - } - } - // This should be more precise, but due to difficulties in getting spans - // from PipelineData::ListStream, this is as it is. - _ => Err(ShellError::UnsupportedInput { - msg: "non-string input".into(), - input: "value originates from here".into(), - msg_span: head, - input_span: input.span().unwrap_or(head), - }), + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let encoding: Spanned = call.req_const(working_set, 0)?; + let ignore_errors = call.has_flag_const(working_set, "ignore-errors")?; + run(call, input, encoding, ignore_errors) + } +} + +fn run( + call: &Call, + input: PipelineData, + encoding: Spanned, + ignore_errors: bool, +) -> Result { + let head = call.head; + + match input { + PipelineData::ByteStream(stream, ..) => { + let span = stream.span(); + let s = stream.into_string()?; + super::encoding::encode(head, encoding, &s, span, ignore_errors) + .map(|val| val.into_pipeline_data()) } + PipelineData::Value(v, ..) => { + let span = v.span(); + match v { + Value::String { val: s, .. } => { + super::encoding::encode(head, encoding, &s, span, ignore_errors) + .map(|val| val.into_pipeline_data()) + } + Value::Error { error, .. } => Err(*error), + _ => Err(ShellError::OnlySupportsThisInputType { + exp_input_type: "string".into(), + wrong_type: v.get_type().to_string(), + dst_span: head, + src_span: v.span(), + }), + } + } + // This should be more precise, but due to difficulties in getting spans + // from PipelineData::ListStream, this is as it is. + _ => Err(ShellError::UnsupportedInput { + msg: "non-string input".into(), + input: "value originates from here".into(), + msg_span: head, + input_span: input.span().unwrap_or(head), + }), } } diff --git a/crates/nu-command/src/strings/encode_decode/encode_base64.rs b/crates/nu-command/src/strings/encode_decode/encode_base64.rs index 090941b76b..b01530b107 100644 --- a/crates/nu-command/src/strings/encode_decode/encode_base64.rs +++ b/crates/nu-command/src/strings/encode_decode/encode_base64.rs @@ -1,4 +1,4 @@ -use super::base64::{operate, ActionType, CHARACTER_SET_DESC}; +use super::base64::{operate, ActionType, Base64CommandArguments, CHARACTER_SET_DESC}; use nu_engine::command_prelude::*; #[derive(Clone)] @@ -70,6 +70,10 @@ impl Command for EncodeBase64 { ] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -77,7 +81,34 @@ impl Command for EncodeBase64 { call: &Call, input: PipelineData, ) -> Result { - operate(ActionType::Encode, engine_state, stack, call, input) + let character_set: Option> = + call.get_flag(engine_state, stack, "character-set")?; + let binary = call.has_flag(engine_state, stack, "binary")?; + let cell_paths: Vec = call.rest(engine_state, stack, 0)?; + let args = Base64CommandArguments { + action_type: ActionType::Encode, + binary, + character_set, + }; + operate(engine_state, call, input, cell_paths, args) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let character_set: Option> = + call.get_flag_const(working_set, "character-set")?; + let binary = call.has_flag_const(working_set, "binary")?; + let cell_paths: Vec = call.rest_const(working_set, 0)?; + let args = Base64CommandArguments { + action_type: ActionType::Encode, + binary, + character_set, + }; + operate(working_set.permanent(), call, input, cell_paths, args) } } diff --git a/crates/nu-command/src/strings/format/date.rs b/crates/nu-command/src/strings/format/date.rs index 2c82eb7541..dd005b9216 100644 --- a/crates/nu-command/src/strings/format/date.rs +++ b/crates/nu-command/src/strings/format/date.rs @@ -27,7 +27,7 @@ impl Command for FormatDate { SyntaxShape::String, "The desired format date.", ) - .category(Category::Date) + .category(Category::Strings) } fn usage(&self) -> &str { @@ -38,36 +38,6 @@ impl Command for FormatDate { vec!["fmt", "strftime"] } - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - let head = call.head; - if call.has_flag(engine_state, stack, "list")? { - return Ok(PipelineData::Value( - generate_strftime_list(head, false), - None, - )); - } - - let format = call.opt::>(engine_state, stack, 0)?; - - // This doesn't match explicit nulls - if matches!(input, PipelineData::Empty) { - return Err(ShellError::PipelineEmpty { dst_span: head }); - } - input.map( - move |value| match &format { - Some(format) => format_helper(value, format.item.as_str(), format.span, head), - None => format_helper_rfc2822(value, head), - }, - engine_state.ctrlc.clone(), - ) - } - fn examples(&self) -> Vec { vec![ Example { @@ -104,6 +74,61 @@ impl Command for FormatDate { }, ] } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let list = call.has_flag(engine_state, stack, "list")?; + let format = call.opt::>(engine_state, stack, 0)?; + run(engine_state, call, input, list, format) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let list = call.has_flag_const(working_set, "list")?; + let format = call.opt_const::>(working_set, 0)?; + run(working_set.permanent(), call, input, list, format) + } +} + +fn run( + engine_state: &EngineState, + call: &Call, + input: PipelineData, + list: bool, + format: Option>, +) -> Result { + let head = call.head; + if list { + return Ok(PipelineData::Value( + generate_strftime_list(head, false), + None, + )); + } + + // This doesn't match explicit nulls + if matches!(input, PipelineData::Empty) { + return Err(ShellError::PipelineEmpty { dst_span: head }); + } + input.map( + move |value| match &format { + Some(format) => format_helper(value, format.item.as_str(), format.span, head), + None => format_helper_rfc2822(value, head), + }, + engine_state.signals(), + ) } fn format_from(date_time: DateTime, formatter: &str, span: Span) -> Value diff --git a/crates/nu-command/src/strings/format/duration.rs b/crates/nu-command/src/strings/format/duration.rs index ad6583cec0..fbbe192048 100644 --- a/crates/nu-command/src/strings/format/duration.rs +++ b/crates/nu-command/src/strings/format/duration.rs @@ -53,6 +53,10 @@ impl Command for FormatDuration { vec!["convert", "display", "pattern", "human readable"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -77,7 +81,34 @@ impl Command for FormatDuration { arg, input, call.head, - engine_state.ctrlc.clone(), + engine_state.signals(), + ) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let format_value = call + .req_const::(working_set, 0)? + .coerce_into_string()? + .to_ascii_lowercase(); + let cell_paths: Vec = call.rest_const(working_set, 1)?; + let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); + let float_precision = working_set.permanent().config.float_precision as usize; + let arg = Arguments { + format_value, + float_precision, + cell_paths, + }; + operate( + format_value_impl, + arg, + input, + call.head, + working_set.permanent().signals(), ) } diff --git a/crates/nu-command/src/strings/format/filesize.rs b/crates/nu-command/src/strings/format/filesize.rs index b54dc92f6d..63865ac2ac 100644 --- a/crates/nu-command/src/strings/format/filesize.rs +++ b/crates/nu-command/src/strings/format/filesize.rs @@ -1,6 +1,6 @@ use nu_cmd_base::input_handler::{operate, CmdArgument}; use nu_engine::command_prelude::*; -use nu_protocol::format_filesize; +use nu_protocol::{engine::StateWorkingSet, format_filesize}; struct Arguments { format_value: String, @@ -50,6 +50,10 @@ impl Command for FormatFilesize { vec!["convert", "display", "pattern", "human readable"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -72,7 +76,32 @@ impl Command for FormatFilesize { arg, input, call.head, - engine_state.ctrlc.clone(), + engine_state.signals(), + ) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let format_value = call + .req_const::(working_set, 0)? + .coerce_into_string()? + .to_ascii_lowercase(); + let cell_paths: Vec = call.rest_const(working_set, 1)?; + let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); + let arg = Arguments { + format_value, + cell_paths, + }; + operate( + format_value_impl, + arg, + input, + call.head, + working_set.permanent().signals(), ) } diff --git a/crates/nu-command/src/strings/guess_width.rs b/crates/nu-command/src/strings/guess_width.rs index 59cfbcb2cf..8c758516bd 100644 --- a/crates/nu-command/src/strings/guess_width.rs +++ b/crates/nu-command/src/strings/guess_width.rs @@ -72,7 +72,9 @@ impl GuessWidth { let mut rows = Vec::new(); while let Ok(columns) = self.read() { - rows.push(columns); + if !columns.is_empty() { + rows.push(columns); + } } rows } @@ -175,34 +177,47 @@ fn separator_position(lr: &[char], p: usize, pos: &[usize], n: usize) -> usize { fn split(line: &str, pos: &[usize], trim_space: bool) -> Vec { let mut n = 0; - let mut start = 0; + let mut start_char = 0; let mut columns = Vec::with_capacity(pos.len() + 1); - let lr: Vec = line.chars().collect(); + let (line_char_boundaries, line_chars): (Vec, Vec) = line.char_indices().unzip(); let mut w = 0; - for p in 0..lr.len() { + if line_chars.is_empty() || line_chars.iter().all(|&c| c.is_whitespace()) { + // current line is completely empty, or only filled with whitespace + return Vec::new(); + } else if !pos.is_empty() + && line_chars.iter().all(|&c| !c.is_whitespace()) + && pos[0] < UnicodeWidthStr::width(line) + { + // we have more than 1 column in the input, but the current line has no whitespace, + // and it is longer than the first detected column separation position + // this indicates some kind of decoration line. let's skip it + return Vec::new(); + } + + for p in 0..line_char_boundaries.len() { if pos.is_empty() || n > pos.len() - 1 { - start = p; + start_char = p; break; } if pos[n] <= w { - let end = separator_position(&lr, p, pos, n); - if start > end { + let end_char = separator_position(&line_chars, p, pos, n); + if start_char > end_char { break; } - let col = &line[start..end]; + let col = &line[line_char_boundaries[start_char]..line_char_boundaries[end_char]]; let col = if trim_space { col.trim() } else { col }; columns.push(col.to_string()); n += 1; - start = end; + start_char = end_char; } - w += UnicodeWidthStr::width(lr[p].to_string().as_str()); + w += UnicodeWidthStr::width(line_chars[p].to_string().as_str()); } // add last part. - let col = &line[start..]; + let col = &line[line_char_boundaries[start_char]..]; let col = if trim_space { col.trim() } else { col }; columns.push(col.to_string()); columns @@ -423,6 +438,162 @@ D: 104792064 17042676 87749388 17% /d"; assert_eq!(got, want); } + #[test] + fn test_guess_width_multibyte() { + let input = "A… B\nC… D"; + let r = Box::new(std::io::BufReader::new(input.as_bytes())) as Box; + let reader = std::io::BufReader::new(r); + + let mut guess_width = GuessWidth { + reader, + pos: Vec::new(), + pre_lines: Vec::new(), + pre_count: 0, + limit_split: 0, + }; + + let want = vec![vec!["A…", "B"], vec!["C…", "D"]]; + let got = guess_width.read_all(); + assert_eq!(got, want); + } + + #[test] + fn test_guess_width_combining_diacritical_marks() { + let input = "Name Surname +Ștefan Țincu "; + + let r = Box::new(std::io::BufReader::new(input.as_bytes())) as Box; + let reader = std::io::BufReader::new(r); + + let mut guess_width = GuessWidth { + reader, + pos: Vec::new(), + pre_lines: Vec::new(), + pre_count: 0, + limit_split: 0, + }; + + let want = vec![vec!["Name", "Surname"], vec!["Ștefan", "Țincu"]]; + let got = guess_width.read_all(); + assert_eq!(got, want); + } + + #[test] + fn test_guess_width_single_column() { + let input = "A + +B + +C"; + + let r = Box::new(std::io::BufReader::new(input.as_bytes())) as Box; + let reader = std::io::BufReader::new(r); + + let mut guess_width = GuessWidth { + reader, + pos: Vec::new(), + pre_lines: Vec::new(), + pre_count: 0, + limit_split: 0, + }; + + let want = vec![vec!["A"], vec!["B"], vec!["C"]]; + let got = guess_width.read_all(); + assert_eq!(got, want); + } + + #[test] + fn test_guess_width_row_without_whitespace() { + let input = "A B C D +------- +E F G H"; + + let r = Box::new(std::io::BufReader::new(input.as_bytes())) as Box; + let reader = std::io::BufReader::new(r); + + let mut guess_width = GuessWidth { + reader, + pos: Vec::new(), + pre_lines: Vec::new(), + pre_count: 0, + limit_split: 0, + }; + + let want = vec![vec!["A", "B", "C", "D"], vec!["E", "F", "G", "H"]]; + let got = guess_width.read_all(); + assert_eq!(got, want); + } + + #[test] + fn test_guess_width_row_with_single_column() { + let input = "A B C D +E +F G H I"; + + let r = Box::new(std::io::BufReader::new(input.as_bytes())) as Box; + let reader = std::io::BufReader::new(r); + + let mut guess_width = GuessWidth { + reader, + pos: Vec::new(), + pre_lines: Vec::new(), + pre_count: 0, + limit_split: 0, + }; + + let want = vec![ + vec!["A", "B", "C", "D"], + vec!["E"], + vec!["F", "G", "H", "I"], + ]; + let got = guess_width.read_all(); + assert_eq!(got, want); + } + + #[test] + fn test_guess_width_empty_row() { + let input = "A B C D + +E F G H"; + + let r = Box::new(std::io::BufReader::new(input.as_bytes())) as Box; + let reader = std::io::BufReader::new(r); + + let mut guess_width = GuessWidth { + reader, + pos: Vec::new(), + pre_lines: Vec::new(), + pre_count: 0, + limit_split: 0, + }; + + let want = vec![vec!["A", "B", "C", "D"], vec!["E", "F", "G", "H"]]; + let got = guess_width.read_all(); + assert_eq!(got, want); + } + + #[test] + fn test_guess_width_row_with_only_whitespace() { + let input = "A B C D + +E F G H"; + + let r = Box::new(std::io::BufReader::new(input.as_bytes())) as Box; + let reader = std::io::BufReader::new(r); + + let mut guess_width = GuessWidth { + reader, + pos: Vec::new(), + pre_lines: Vec::new(), + pre_count: 0, + limit_split: 0, + }; + + let want = vec![vec!["A", "B", "C", "D"], vec!["E", "F", "G", "H"]]; + let got = guess_width.read_all(); + assert_eq!(got, want); + } + #[test] fn test_to_table() { let lines = vec![ diff --git a/crates/nu-command/src/strings/mod.rs b/crates/nu-command/src/strings/mod.rs index d1ebf540e5..8b5af2dec4 100644 --- a/crates/nu-command/src/strings/mod.rs +++ b/crates/nu-command/src/strings/mod.rs @@ -17,8 +17,7 @@ pub use str_::*; use nu_engine::CallExt; use nu_protocol::{ - ast::Call, - engine::{EngineState, Stack, StateWorkingSet}, + engine::{Call, EngineState, Stack, StateWorkingSet}, ShellError, }; diff --git a/crates/nu-command/src/strings/parse.rs b/crates/nu-command/src/strings/parse.rs index bc70d4679c..9bf31de73c 100644 --- a/crates/nu-command/src/strings/parse.rs +++ b/crates/nu-command/src/strings/parse.rs @@ -1,10 +1,7 @@ use fancy_regex::{Captures, Regex}; use nu_engine::command_prelude::*; -use nu_protocol::ListStream; -use std::{ - collections::VecDeque, - sync::{atomic::AtomicBool, Arc}, -}; +use nu_protocol::{engine::StateWorkingSet, ListStream, Signals}; +use std::collections::VecDeque; #[derive(Clone)] pub struct Parse; @@ -99,6 +96,10 @@ impl Command for Parse { ] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -106,19 +107,31 @@ impl Command for Parse { call: &Call, input: PipelineData, ) -> Result { - operate(engine_state, stack, call, input) + let pattern: Spanned = call.req(engine_state, stack, 0)?; + let regex: bool = call.has_flag(engine_state, stack, "regex")?; + operate(engine_state, pattern, regex, call, input) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let pattern: Spanned = call.req_const(working_set, 0)?; + let regex: bool = call.has_flag_const(working_set, "regex")?; + operate(working_set.permanent(), pattern, regex, call, input) } } fn operate( engine_state: &EngineState, - stack: &mut Stack, + pattern: Spanned, + regex: bool, call: &Call, input: PipelineData, ) -> Result { let head = call.head; - let pattern: Spanned = call.req(engine_state, stack, 0)?; - let regex: bool = call.has_flag(engine_state, stack, "regex")?; let pattern_item = pattern.item; let pattern_span = pattern.span; @@ -147,8 +160,6 @@ fn operate( }) .collect::>(); - let ctrlc = engine_state.ctrlc.clone(); - match input { PipelineData::Empty => Ok(PipelineData::Empty), PipelineData::Value(value, ..) => match value { @@ -176,10 +187,10 @@ fn operate( columns, iter, span: head, - ctrlc, + signals: engine_state.signals().clone(), }; - Ok(ListStream::new(iter, head, None).into()) + Ok(ListStream::new(iter, head, Signals::empty()).into()) } value => Err(ShellError::PipelineMismatch { exp_input_type: "string".into(), @@ -204,7 +215,7 @@ fn operate( columns, iter, span: head, - ctrlc, + signals: engine_state.signals().clone(), } }) .into()), @@ -216,10 +227,10 @@ fn operate( columns, iter: lines, span: head, - ctrlc, + signals: engine_state.signals().clone(), }; - Ok(ListStream::new(iter, head, None).into()) + Ok(ListStream::new(iter, head, Signals::empty()).into()) } else { Ok(PipelineData::Empty) } @@ -286,7 +297,7 @@ struct ParseIter>> { columns: Vec, iter: I, span: Span, - ctrlc: Option>, + signals: Signals, } impl>> ParseIter { @@ -304,7 +315,7 @@ impl>> Iterator for ParseIter { fn next(&mut self) -> Option { loop { - if nu_utils::ctrl_c::was_pressed(&self.ctrlc) { + if self.signals.interrupted() { return None; } diff --git a/crates/nu-command/src/strings/split/chars.rs b/crates/nu-command/src/strings/split/chars.rs index 625915e76d..370df262ea 100644 --- a/crates/nu-command/src/strings/split/chars.rs +++ b/crates/nu-command/src/strings/split/chars.rs @@ -1,5 +1,6 @@ -use crate::grapheme_flags; +use crate::{grapheme_flags, grapheme_flags_const}; use nu_engine::command_prelude::*; + use unicode_segmentation::UnicodeSegmentation; #[derive(Clone)] @@ -88,6 +89,10 @@ impl Command for SubCommand { ] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -95,22 +100,31 @@ impl Command for SubCommand { call: &Call, input: PipelineData, ) -> Result { - split_chars(engine_state, stack, call, input) + let graphemes = grapheme_flags(engine_state, stack, call)?; + split_chars(engine_state, call, input, graphemes) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let graphemes = grapheme_flags_const(working_set, call)?; + split_chars(working_set.permanent(), call, input, graphemes) } } fn split_chars( engine_state: &EngineState, - stack: &mut Stack, call: &Call, input: PipelineData, + graphemes: bool, ) -> Result { let span = call.head; - - let graphemes = grapheme_flags(engine_state, stack, call)?; input.map( move |x| split_chars_helper(&x, span, graphemes), - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-command/src/strings/split/column.rs b/crates/nu-command/src/strings/split/column.rs index d73243322d..540cfabe54 100644 --- a/crates/nu-command/src/strings/split/column.rs +++ b/crates/nu-command/src/strings/split/column.rs @@ -43,16 +43,6 @@ impl Command for SubCommand { vec!["separate", "divide", "regex"] } - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - split_column(engine_state, stack, call, input) - } - fn examples(&self) -> Vec { vec![ Example { @@ -103,36 +93,84 @@ impl Command for SubCommand { }, ] } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let separator: Spanned = call.req(engine_state, stack, 0)?; + let rest: Vec> = call.rest(engine_state, stack, 1)?; + let collapse_empty = call.has_flag(engine_state, stack, "collapse-empty")?; + let has_regex = call.has_flag(engine_state, stack, "regex")?; + + let args = Arguments { + separator, + rest, + collapse_empty, + has_regex, + }; + split_column(engine_state, call, input, args) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let separator: Spanned = call.req_const(working_set, 0)?; + let rest: Vec> = call.rest_const(working_set, 1)?; + let collapse_empty = call.has_flag_const(working_set, "collapse-empty")?; + let has_regex = call.has_flag_const(working_set, "regex")?; + + let args = Arguments { + separator, + rest, + collapse_empty, + has_regex, + }; + split_column(working_set.permanent(), call, input, args) + } +} + +struct Arguments { + separator: Spanned, + rest: Vec>, + collapse_empty: bool, + has_regex: bool, } fn split_column( engine_state: &EngineState, - stack: &mut Stack, call: &Call, input: PipelineData, + args: Arguments, ) -> Result { let name_span = call.head; - let separator: Spanned = call.req(engine_state, stack, 0)?; - let rest: Vec> = call.rest(engine_state, stack, 1)?; - let collapse_empty = call.has_flag(engine_state, stack, "collapse-empty")?; - - let regex = if call.has_flag(engine_state, stack, "regex")? { - Regex::new(&separator.item) + let regex = if args.has_regex { + Regex::new(&args.separator.item) } else { - let escaped = regex::escape(&separator.item); + let escaped = regex::escape(&args.separator.item); Regex::new(&escaped) } .map_err(|e| ShellError::GenericError { error: "Error with regular expression".into(), msg: e.to_string(), - span: Some(separator.span), + span: Some(args.separator.span), help: None, inner: vec![], })?; input.flat_map( - move |x| split_column_helper(&x, ®ex, &rest, collapse_empty, name_span), - engine_state.ctrlc.clone(), + move |x| split_column_helper(&x, ®ex, &args.rest, args.collapse_empty, name_span), + engine_state.signals(), ) } diff --git a/crates/nu-command/src/strings/split/list.rs b/crates/nu-command/src/strings/split/list.rs index 470ba12ec4..eb874841a2 100644 --- a/crates/nu-command/src/strings/split/list.rs +++ b/crates/nu-command/src/strings/split/list.rs @@ -36,16 +36,6 @@ impl Command for SubCommand { vec!["separate", "divide", "regex"] } - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - split_list(engine_state, stack, call, input) - } - fn examples(&self) -> Vec { vec![ Example { @@ -145,6 +135,33 @@ impl Command for SubCommand { }, ] } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let has_regex = call.has_flag(engine_state, stack, "regex")?; + let separator: Value = call.req(engine_state, stack, 0)?; + split_list(engine_state, call, input, has_regex, separator) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let has_regex = call.has_flag_const(working_set, "regex")?; + let separator: Value = call.req_const(working_set, 0)?; + split_list(working_set.permanent(), call, input, has_regex, separator) + } } enum Matcher { @@ -188,19 +205,17 @@ impl Matcher { fn split_list( engine_state: &EngineState, - stack: &mut Stack, call: &Call, input: PipelineData, + has_regex: bool, + separator: Value, ) -> Result { - let separator: Value = call.req(engine_state, stack, 0)?; let mut temp_list = Vec::new(); let mut returned_list = Vec::new(); - let matcher = Matcher::new(call.has_flag(engine_state, stack, "regex")?, separator)?; + let matcher = Matcher::new(has_regex, separator)?; for val in input { - if nu_utils::ctrl_c::was_pressed(&engine_state.ctrlc) { - break; - } + engine_state.signals().check(call.head)?; if matcher.compare(&val)? { if !temp_list.is_empty() { diff --git a/crates/nu-command/src/strings/split/row.rs b/crates/nu-command/src/strings/split/row.rs index 8ee22b213c..1f427a06e0 100644 --- a/crates/nu-command/src/strings/split/row.rs +++ b/crates/nu-command/src/strings/split/row.rs @@ -43,16 +43,6 @@ impl Command for SubCommand { vec!["separate", "divide", "regex"] } - fn run( - &self, - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, - input: PipelineData, - ) -> Result { - split_row(engine_state, stack, call, input) - } - fn examples(&self) -> Vec { vec![ Example { @@ -109,33 +99,78 @@ impl Command for SubCommand { }, ] } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let separator: Spanned = call.req(engine_state, stack, 0)?; + let max_split: Option = call.get_flag(engine_state, stack, "number")?; + let has_regex = call.has_flag(engine_state, stack, "regex")?; + + let args = Arguments { + separator, + max_split, + has_regex, + }; + split_row(engine_state, call, input, args) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let separator: Spanned = call.req_const(working_set, 0)?; + let max_split: Option = call.get_flag_const(working_set, "number")?; + let has_regex = call.has_flag_const(working_set, "regex")?; + + let args = Arguments { + separator, + max_split, + has_regex, + }; + split_row(working_set.permanent(), call, input, args) + } +} + +struct Arguments { + has_regex: bool, + separator: Spanned, + max_split: Option, } fn split_row( engine_state: &EngineState, - stack: &mut Stack, call: &Call, input: PipelineData, + args: Arguments, ) -> Result { let name_span = call.head; - let separator: Spanned = call.req(engine_state, stack, 0)?; - let regex = if call.has_flag(engine_state, stack, "regex")? { - Regex::new(&separator.item) + let regex = if args.has_regex { + Regex::new(&args.separator.item) } else { - let escaped = regex::escape(&separator.item); + let escaped = regex::escape(&args.separator.item); Regex::new(&escaped) } .map_err(|e| ShellError::GenericError { error: "Error with regular expression".into(), msg: e.to_string(), - span: Some(separator.span), + span: Some(args.separator.span), help: None, inner: vec![], })?; - let max_split: Option = call.get_flag(engine_state, stack, "number")?; input.flat_map( - move |x| split_row_helper(&x, ®ex, max_split, name_span), - engine_state.ctrlc.clone(), + move |x| split_row_helper(&x, ®ex, args.max_split, name_span), + engine_state.signals(), ) } diff --git a/crates/nu-command/src/strings/split/words.rs b/crates/nu-command/src/strings/split/words.rs index 17b68bd44f..6cb5562a70 100644 --- a/crates/nu-command/src/strings/split/words.rs +++ b/crates/nu-command/src/strings/split/words.rs @@ -1,4 +1,4 @@ -use crate::grapheme_flags; +use crate::{grapheme_flags, grapheme_flags_const}; use fancy_regex::Regex; use nu_engine::command_prelude::*; @@ -96,6 +96,10 @@ impl Command for SubCommand { ] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -103,41 +107,77 @@ impl Command for SubCommand { call: &Call, input: PipelineData, ) -> Result { - split_words(engine_state, stack, call, input) + let word_length: Option = call.get_flag(engine_state, stack, "min-word-length")?; + let has_grapheme = call.has_flag(engine_state, stack, "grapheme-clusters")?; + let has_utf8 = call.has_flag(engine_state, stack, "utf-8-bytes")?; + let graphemes = grapheme_flags(engine_state, stack, call)?; + + let args = Arguments { + word_length, + has_grapheme, + has_utf8, + graphemes, + }; + split_words(engine_state, call, input, args) } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let word_length: Option = call.get_flag_const(working_set, "min-word-length")?; + let has_grapheme = call.has_flag_const(working_set, "grapheme-clusters")?; + let has_utf8 = call.has_flag_const(working_set, "utf-8-bytes")?; + let graphemes = grapheme_flags_const(working_set, call)?; + + let args = Arguments { + word_length, + has_grapheme, + has_utf8, + graphemes, + }; + split_words(working_set.permanent(), call, input, args) + } +} + +struct Arguments { + word_length: Option, + has_grapheme: bool, + has_utf8: bool, + graphemes: bool, } fn split_words( engine_state: &EngineState, - stack: &mut Stack, call: &Call, input: PipelineData, + args: Arguments, ) -> Result { let span = call.head; // let ignore_hyphenated = call.has_flag(engine_state, stack, "ignore-hyphenated")?; // let ignore_apostrophes = call.has_flag(engine_state, stack, "ignore-apostrophes")?; // let ignore_punctuation = call.has_flag(engine_state, stack, "ignore-punctuation")?; - let word_length: Option = call.get_flag(engine_state, stack, "min-word-length")?; - if word_length.is_none() { - if call.has_flag(engine_state, stack, "grapheme-clusters")? { + if args.word_length.is_none() { + if args.has_grapheme { return Err(ShellError::IncompatibleParametersSingle { msg: "--grapheme-clusters (-g) requires --min-word-length (-l)".to_string(), span, }); } - if call.has_flag(engine_state, stack, "utf-8-bytes")? { + if args.has_utf8 { return Err(ShellError::IncompatibleParametersSingle { msg: "--utf-8-bytes (-b) requires --min-word-length (-l)".to_string(), span, }); } } - let graphemes = grapheme_flags(engine_state, stack, call)?; input.map( - move |x| split_words_helper(&x, word_length, span, graphemes), - engine_state.ctrlc.clone(), + move |x| split_words_helper(&x, args.word_length, span, args.graphemes), + engine_state.signals(), ) } diff --git a/crates/nu-command/src/strings/str_/case/capitalize.rs b/crates/nu-command/src/strings/str_/case/capitalize.rs index 82f0d102e6..862ca127c2 100644 --- a/crates/nu-command/src/strings/str_/case/capitalize.rs +++ b/crates/nu-command/src/strings/str_/case/capitalize.rs @@ -36,6 +36,10 @@ impl Command for SubCommand { vec!["convert", "style", "caps", "upper"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -43,7 +47,18 @@ impl Command for SubCommand { call: &Call, input: PipelineData, ) -> Result { - operate(engine_state, stack, call, input) + let column_paths: Vec = call.rest(engine_state, stack, 0)?; + operate(engine_state, call, input, column_paths) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let column_paths: Vec = call.rest_const(working_set, 0)?; + operate(working_set.permanent(), call, input, column_paths) } fn examples(&self) -> Vec { @@ -72,12 +87,11 @@ impl Command for SubCommand { fn operate( engine_state: &EngineState, - stack: &mut Stack, call: &Call, input: PipelineData, + column_paths: Vec, ) -> Result { let head = call.head; - let column_paths: Vec = call.rest(engine_state, stack, 0)?; input.map( move |v| { if column_paths.is_empty() { @@ -94,7 +108,7 @@ fn operate( ret } }, - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-command/src/strings/str_/case/downcase.rs b/crates/nu-command/src/strings/str_/case/downcase.rs index 7fa4785499..0493a663aa 100644 --- a/crates/nu-command/src/strings/str_/case/downcase.rs +++ b/crates/nu-command/src/strings/str_/case/downcase.rs @@ -36,6 +36,10 @@ impl Command for SubCommand { vec!["lower case", "lowercase"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -43,7 +47,18 @@ impl Command for SubCommand { call: &Call, input: PipelineData, ) -> Result { - operate(engine_state, stack, call, input) + let column_paths: Vec = call.rest(engine_state, stack, 0)?; + operate(engine_state, call, input, column_paths) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let column_paths: Vec = call.rest_const(working_set, 0)?; + operate(working_set.permanent(), call, input, column_paths) } fn examples(&self) -> Vec { @@ -80,12 +95,11 @@ impl Command for SubCommand { fn operate( engine_state: &EngineState, - stack: &mut Stack, call: &Call, input: PipelineData, + column_paths: Vec, ) -> Result { let head = call.head; - let column_paths: Vec = call.rest(engine_state, stack, 0)?; input.map( move |v| { if column_paths.is_empty() { @@ -102,7 +116,7 @@ fn operate( ret } }, - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-command/src/strings/str_/case/mod.rs b/crates/nu-command/src/strings/str_/case/mod.rs index 32b50c4bc6..3390ff097e 100644 --- a/crates/nu-command/src/strings/str_/case/mod.rs +++ b/crates/nu-command/src/strings/str_/case/mod.rs @@ -38,7 +38,7 @@ where case_operation, cell_paths, }; - general_operate(action, args, input, call.head, engine_state.ctrlc.clone()) + general_operate(action, args, input, call.head, engine_state.signals()) } fn action(input: &Value, args: &Arguments, head: Span) -> Value diff --git a/crates/nu-command/src/strings/str_/case/upcase.rs b/crates/nu-command/src/strings/str_/case/upcase.rs index 222c9eeab4..557239ec91 100644 --- a/crates/nu-command/src/strings/str_/case/upcase.rs +++ b/crates/nu-command/src/strings/str_/case/upcase.rs @@ -36,6 +36,10 @@ impl Command for SubCommand { vec!["uppercase", "upper case"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -43,7 +47,18 @@ impl Command for SubCommand { call: &Call, input: PipelineData, ) -> Result { - operate(engine_state, stack, call, input) + let column_paths: Vec = call.rest(engine_state, stack, 0)?; + operate(engine_state, call, input, column_paths) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let column_paths: Vec = call.rest_const(working_set, 0)?; + operate(working_set.permanent(), call, input, column_paths) } fn examples(&self) -> Vec { @@ -57,12 +72,11 @@ impl Command for SubCommand { fn operate( engine_state: &EngineState, - stack: &mut Stack, call: &Call, input: PipelineData, + column_paths: Vec, ) -> Result { let head = call.head; - let column_paths: Vec = call.rest(engine_state, stack, 0)?; input.map( move |v| { if column_paths.is_empty() { @@ -79,7 +93,7 @@ fn operate( ret } }, - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-command/src/strings/str_/contains.rs b/crates/nu-command/src/strings/str_/contains.rs index bc1d5bcf11..27d8a8e313 100644 --- a/crates/nu-command/src/strings/str_/contains.rs +++ b/crates/nu-command/src/strings/str_/contains.rs @@ -10,7 +10,6 @@ struct Arguments { substring: String, cell_paths: Option>, case_insensitive: bool, - not_contain: bool, } impl CmdArgument for Arguments { @@ -40,7 +39,6 @@ impl Command for SubCommand { "For a data structure input, check strings at the given cell paths, and replace with result.", ) .switch("ignore-case", "search is case insensitive", Some('i')) - .switch("not", "DEPRECATED OPTION: does not contain", Some('n')) .category(Category::Strings) } @@ -52,6 +50,10 @@ impl Command for SubCommand { vec!["substring", "match", "find", "search"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -59,9 +61,25 @@ impl Command for SubCommand { call: &Call, input: PipelineData, ) -> Result { - if call.has_flag(engine_state, stack, "not")? { + let cell_paths: Vec = call.rest(engine_state, stack, 1)?; + let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); + let args = Arguments { + substring: call.req::(engine_state, stack, 0)?, + cell_paths, + case_insensitive: call.has_flag(engine_state, stack, "ignore-case")?, + }; + operate(action, args, input, call.head, engine_state.signals()) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + if call.has_flag_const(working_set, "not")? { nu_protocol::report_error_new( - engine_state, + working_set.permanent(), &ShellError::GenericError { error: "Deprecated option".into(), msg: "`str contains --not {string}` is deprecated and will be removed in 0.95." @@ -73,15 +91,20 @@ impl Command for SubCommand { ); } - let cell_paths: Vec = call.rest(engine_state, stack, 1)?; + let cell_paths: Vec = call.rest_const(working_set, 1)?; let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); let args = Arguments { - substring: call.req::(engine_state, stack, 0)?, + substring: call.req_const::(working_set, 0)?, cell_paths, - case_insensitive: call.has_flag(engine_state, stack, "ignore-case")?, - not_contain: call.has_flag(engine_state, stack, "not")?, + case_insensitive: call.has_flag_const(working_set, "ignore-case")?, }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate( + action, + args, + input, + call.head, + working_set.permanent().signals(), + ) } fn examples(&self) -> Vec { @@ -142,7 +165,6 @@ fn action( input: &Value, Arguments { case_insensitive, - not_contain, substring, .. }: &Arguments, @@ -150,23 +172,11 @@ fn action( ) -> Value { match input { Value::String { val, .. } => Value::bool( - match case_insensitive { - true => { - if *not_contain { - !val.to_folded_case() - .contains(substring.to_folded_case().as_str()) - } else { - val.to_folded_case() - .contains(substring.to_folded_case().as_str()) - } - } - false => { - if *not_contain { - !val.contains(substring) - } else { - val.contains(substring) - } - } + if *case_insensitive { + val.to_folded_case() + .contains(substring.to_folded_case().as_str()) + } else { + val.contains(substring) }, head, ), diff --git a/crates/nu-command/src/strings/str_/deunicode.rs b/crates/nu-command/src/strings/str_/deunicode.rs new file mode 100644 index 0000000000..ef732571f9 --- /dev/null +++ b/crates/nu-command/src/strings/str_/deunicode.rs @@ -0,0 +1,98 @@ +use deunicode::deunicode; +use nu_cmd_base::input_handler::{operate, CellPathOnlyArgs}; +use nu_engine::command_prelude::*; +use nu_protocol::engine::StateWorkingSet; + +#[derive(Clone)] +pub struct SubCommand; + +impl Command for SubCommand { + fn name(&self) -> &str { + "str deunicode" + } + + fn signature(&self) -> Signature { + Signature::build("str deunicode") + .input_output_types(vec![(Type::String, Type::String)]) + .category(Category::Strings) + } + + fn usage(&self) -> &str { + "Convert Unicode string to pure ASCII." + } + + fn search_terms(&self) -> Vec<&str> { + vec!["convert", "ascii"] + } + + fn is_const(&self) -> bool { + true + } + + fn run( + &self, + engine_state: &EngineState, + stack: &mut Stack, + call: &Call, + input: PipelineData, + ) -> Result { + let cell_paths: Vec = call.rest(engine_state, stack, 0)?; + let args = CellPathOnlyArgs::from(cell_paths); + + operate(action, args, input, call.head, engine_state.signals()) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let cell_paths: Vec = call.rest_const(working_set, 0)?; + let args = CellPathOnlyArgs::from(cell_paths); + + operate( + action, + args, + input, + call.head, + working_set.permanent().signals(), + ) + } + + fn examples(&self) -> Vec { + vec![Example { + description: "deunicode a string", + example: "'A…B' | str deunicode", + result: Some(Value::test_string("A...B")), + }] + } +} + +fn action(input: &Value, _args: &CellPathOnlyArgs, head: Span) -> Value { + match input { + Value::String { val, .. } => Value::string(deunicode(val), head), + Value::Error { .. } => input.clone(), + _ => Value::error( + ShellError::OnlySupportsThisInputType { + exp_input_type: "string".into(), + wrong_type: input.get_type().to_string(), + dst_span: head, + src_span: input.span(), + }, + head, + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_examples() { + use crate::test_examples; + + test_examples(SubCommand {}) + } +} diff --git a/crates/nu-command/src/strings/str_/distance.rs b/crates/nu-command/src/strings/str_/distance.rs index aa45ec5c25..9b88f8e3e0 100644 --- a/crates/nu-command/src/strings/str_/distance.rs +++ b/crates/nu-command/src/strings/str_/distance.rs @@ -1,6 +1,6 @@ use nu_cmd_base::input_handler::{operate, CmdArgument}; use nu_engine::command_prelude::*; -use nu_protocol::levenshtein_distance; +use nu_protocol::{engine::StateWorkingSet, levenshtein_distance}; #[derive(Clone)] pub struct SubCommand; @@ -49,6 +49,10 @@ impl Command for SubCommand { vec!["edit", "levenshtein"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -63,7 +67,29 @@ impl Command for SubCommand { compare_string, cell_paths, }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let compare_string: String = call.req_const(working_set, 0)?; + let cell_paths: Vec = call.rest_const(working_set, 1)?; + let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); + let args = Arguments { + compare_string, + cell_paths, + }; + operate( + action, + args, + input, + call.head, + working_set.permanent().signals(), + ) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/strings/str_/ends_with.rs b/crates/nu-command/src/strings/str_/ends_with.rs index 1b06acd880..d4841b74e0 100644 --- a/crates/nu-command/src/strings/str_/ends_with.rs +++ b/crates/nu-command/src/strings/str_/ends_with.rs @@ -50,6 +50,10 @@ impl Command for SubCommand { vec!["suffix", "match", "find", "search"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -64,7 +68,29 @@ impl Command for SubCommand { cell_paths, case_insensitive: call.has_flag(engine_state, stack, "ignore-case")?, }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let cell_paths: Vec = call.rest_const(working_set, 1)?; + let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); + let args = Arguments { + substring: call.req_const::(working_set, 0)?, + cell_paths, + case_insensitive: call.has_flag_const(working_set, "ignore-case")?, + }; + operate( + action, + args, + input, + call.head, + working_set.permanent().signals(), + ) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/strings/str_/expand.rs b/crates/nu-command/src/strings/str_/expand.rs index 2eca970403..b9759ef6a1 100644 --- a/crates/nu-command/src/strings/str_/expand.rs +++ b/crates/nu-command/src/strings/str_/expand.rs @@ -179,6 +179,10 @@ impl Command for SubCommand { ] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -186,32 +190,51 @@ impl Command for SubCommand { call: &Call, input: PipelineData, ) -> Result { - let span = call.head; - if matches!(input, PipelineData::Empty) { - return Err(ShellError::PipelineEmpty { dst_span: span }); - } let is_path = call.has_flag(engine_state, stack, "path")?; - input.map( - move |v| { - let value_span = v.span(); - match v.coerce_into_string() { - Ok(s) => { - let contents = if is_path { s.replace('\\', "\\\\") } else { s }; - str_expand(&contents, span, value_span) - } - Err(_) => Value::error( - ShellError::PipelineMismatch { - exp_input_type: "string".into(), - dst_span: span, - src_span: value_span, - }, - span, - ), - } - }, - engine_state.ctrlc.clone(), - ) + run(call, input, is_path, engine_state) } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let is_path = call.has_flag_const(working_set, "path")?; + run(call, input, is_path, working_set.permanent()) + } +} + +fn run( + call: &Call, + input: PipelineData, + is_path: bool, + engine_state: &EngineState, +) -> Result { + let span = call.head; + if matches!(input, PipelineData::Empty) { + return Err(ShellError::PipelineEmpty { dst_span: span }); + } + input.map( + move |v| { + let value_span = v.span(); + match v.coerce_into_string() { + Ok(s) => { + let contents = if is_path { s.replace('\\', "\\\\") } else { s }; + str_expand(&contents, span, value_span) + } + Err(_) => Value::error( + ShellError::PipelineMismatch { + exp_input_type: "string".into(), + dst_span: span, + src_span: value_span, + }, + span, + ), + } + }, + engine_state.signals(), + ) } fn str_expand(contents: &str, span: Span, value_span: Span) -> Value { diff --git a/crates/nu-command/src/strings/str_/index_of.rs b/crates/nu-command/src/strings/str_/index_of.rs index 457713c2df..a6b06ef9b7 100644 --- a/crates/nu-command/src/strings/str_/index_of.rs +++ b/crates/nu-command/src/strings/str_/index_of.rs @@ -1,10 +1,10 @@ -use crate::grapheme_flags; +use crate::{grapheme_flags, grapheme_flags_const}; use nu_cmd_base::{ input_handler::{operate, CmdArgument}, util, }; use nu_engine::command_prelude::*; -use nu_protocol::Range; +use nu_protocol::{engine::StateWorkingSet, Range}; use unicode_segmentation::UnicodeSegmentation; struct Arguments { @@ -72,6 +72,10 @@ impl Command for SubCommand { vec!["match", "find", "search"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -89,7 +93,32 @@ impl Command for SubCommand { cell_paths, graphemes: grapheme_flags(engine_state, stack, call)?, }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let substring: Spanned = call.req_const(working_set, 0)?; + let cell_paths: Vec = call.rest_const(working_set, 1)?; + let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); + let args = Arguments { + substring: substring.item, + range: call.get_flag_const(working_set, "range")?, + end: call.has_flag_const(working_set, "end")?, + cell_paths, + graphemes: grapheme_flags_const(working_set, call)?, + }; + operate( + action, + args, + input, + call.head, + working_set.permanent().signals(), + ) } fn examples(&self) -> Vec { @@ -405,7 +434,7 @@ mod tests { let range = Range::new( Value::int(0, Span::test_data()), Value::int(1, Span::test_data()), - Value::int(3, Span::test_data()), + Value::int(2, Span::test_data()), RangeInclusion::Inclusive, Span::test_data(), ) diff --git a/crates/nu-command/src/strings/str_/join.rs b/crates/nu-command/src/strings/str_/join.rs index 732434b20f..d297ea4cbc 100644 --- a/crates/nu-command/src/strings/str_/join.rs +++ b/crates/nu-command/src/strings/str_/join.rs @@ -1,4 +1,7 @@ use nu_engine::command_prelude::*; +use nu_protocol::Signals; + +use std::io::Write; #[derive(Clone)] pub struct StrJoin; @@ -31,6 +34,10 @@ impl Command for StrJoin { vec!["collect", "concatenate"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -39,32 +46,17 @@ impl Command for StrJoin { input: PipelineData, ) -> Result { let separator: Option = call.opt(engine_state, stack, 0)?; + run(engine_state, call, input, separator) + } - let config = engine_state.get_config(); - - // let output = input.collect_string(&separator.unwrap_or_default(), &config)?; - // Hmm, not sure what we actually want. - // `to_formatted_string` formats dates as human readable which feels funny. - let mut strings: Vec = vec![]; - - for value in input { - let str = match value { - Value::Error { error, .. } => { - return Err(*error); - } - Value::Date { val, .. } => format!("{val:?}"), - value => value.to_expanded_string("\n", config), - }; - strings.push(str); - } - - let output = if let Some(separator) = separator { - strings.join(&separator) - } else { - strings.join("") - }; - - Ok(Value::string(output, call.head).into_pipeline_data()) + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let separator: Option = call.opt_const(working_set, 0)?; + run(working_set.permanent(), call, input, separator) } fn examples(&self) -> Vec { @@ -83,6 +75,53 @@ impl Command for StrJoin { } } +fn run( + engine_state: &EngineState, + call: &Call, + input: PipelineData, + separator: Option, +) -> Result { + let config = engine_state.config.clone(); + + let span = call.head; + + let metadata = input.metadata(); + let mut iter = input.into_iter(); + let mut first = true; + + let output = ByteStream::from_fn( + span, + Signals::empty(), + ByteStreamType::String, + move |buffer| { + // Write each input to the buffer + if let Some(value) = iter.next() { + // Write the separator if this is not the first + if first { + first = false; + } else if let Some(separator) = &separator { + write!(buffer, "{}", separator)?; + } + + match value { + Value::Error { error, .. } => { + return Err(*error); + } + // Hmm, not sure what we actually want. + // `to_expanded_string` formats dates as human readable which feels funny. + Value::Date { val, .. } => write!(buffer, "{val:?}")?, + value => write!(buffer, "{}", value.to_expanded_string("\n", &config))?, + } + Ok(true) + } else { + Ok(false) + } + }, + ); + + Ok(PipelineData::ByteStream(output, metadata)) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/nu-command/src/strings/str_/length.rs b/crates/nu-command/src/strings/str_/length.rs index 6e2ae4182b..ab0ba8db49 100644 --- a/crates/nu-command/src/strings/str_/length.rs +++ b/crates/nu-command/src/strings/str_/length.rs @@ -1,7 +1,7 @@ use crate::{grapheme_flags, grapheme_flags_const}; use nu_cmd_base::input_handler::{operate, CmdArgument}; use nu_engine::command_prelude::*; -use nu_protocol::engine::StateWorkingSet; + use unicode_segmentation::UnicodeSegmentation; struct Arguments { @@ -130,7 +130,7 @@ fn run( cell_paths: (!cell_paths.is_empty()).then_some(cell_paths), graphemes, }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) } fn action(input: &Value, arg: &Arguments, head: Span) -> Value { diff --git a/crates/nu-command/src/strings/str_/mod.rs b/crates/nu-command/src/strings/str_/mod.rs index fb34ace833..b9290ada2c 100644 --- a/crates/nu-command/src/strings/str_/mod.rs +++ b/crates/nu-command/src/strings/str_/mod.rs @@ -1,5 +1,6 @@ mod case; mod contains; +mod deunicode; mod distance; mod ends_with; mod expand; @@ -15,6 +16,7 @@ mod trim; pub use case::*; pub use contains::SubCommand as StrContains; +pub use deunicode::SubCommand as StrDeunicode; pub use distance::SubCommand as StrDistance; pub use ends_with::SubCommand as StrEndswith; pub use expand::SubCommand as StrExpand; diff --git a/crates/nu-command/src/strings/str_/replace.rs b/crates/nu-command/src/strings/str_/replace.rs index 5d5863e70a..342dd0693e 100644 --- a/crates/nu-command/src/strings/str_/replace.rs +++ b/crates/nu-command/src/strings/str_/replace.rs @@ -73,6 +73,10 @@ impl Command for SubCommand { vec!["search", "shift", "switch", "regex"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -98,7 +102,40 @@ impl Command for SubCommand { no_regex, multiline, }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let find: Spanned = call.req_const(working_set, 0)?; + let replace: Spanned = call.req_const(working_set, 1)?; + let cell_paths: Vec = call.rest_const(working_set, 2)?; + let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); + let literal_replace = call.has_flag_const(working_set, "no-expand")?; + let no_regex = !call.has_flag_const(working_set, "regex")? + && !call.has_flag_const(working_set, "multiline")?; + let multiline = call.has_flag_const(working_set, "multiline")?; + + let args = Arguments { + all: call.has_flag_const(working_set, "all")?, + find, + replace, + cell_paths, + literal_replace, + no_regex, + multiline, + }; + operate( + action, + args, + input, + call.head, + working_set.permanent().signals(), + ) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/strings/str_/reverse.rs b/crates/nu-command/src/strings/str_/reverse.rs index becfd9be50..9a339f7bcc 100644 --- a/crates/nu-command/src/strings/str_/reverse.rs +++ b/crates/nu-command/src/strings/str_/reverse.rs @@ -37,6 +37,10 @@ impl Command for SubCommand { vec!["convert", "inverse", "flip"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -46,7 +50,24 @@ impl Command for SubCommand { ) -> Result { let cell_paths: Vec = call.rest(engine_state, stack, 0)?; let args = CellPathOnlyArgs::from(cell_paths); - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let cell_paths: Vec = call.rest_const(working_set, 0)?; + let args = CellPathOnlyArgs::from(cell_paths); + operate( + action, + args, + input, + call.head, + working_set.permanent().signals(), + ) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/strings/str_/starts_with.rs b/crates/nu-command/src/strings/str_/starts_with.rs index 73396911e2..bac451466c 100644 --- a/crates/nu-command/src/strings/str_/starts_with.rs +++ b/crates/nu-command/src/strings/str_/starts_with.rs @@ -51,6 +51,10 @@ impl Command for SubCommand { vec!["prefix", "match", "find", "search"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -66,7 +70,30 @@ impl Command for SubCommand { cell_paths, case_insensitive: call.has_flag(engine_state, stack, "ignore-case")?, }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let substring: Spanned = call.req_const(working_set, 0)?; + let cell_paths: Vec = call.rest_const(working_set, 1)?; + let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); + let args = Arguments { + substring: substring.item, + cell_paths, + case_insensitive: call.has_flag_const(working_set, "ignore-case")?, + }; + operate( + action, + args, + input, + call.head, + working_set.permanent().signals(), + ) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/strings/str_/stats.rs b/crates/nu-command/src/strings/str_/stats.rs index 20ef35c51f..eb091ae4f6 100644 --- a/crates/nu-command/src/strings/str_/stats.rs +++ b/crates/nu-command/src/strings/str_/stats.rs @@ -1,5 +1,6 @@ use fancy_regex::Regex; use nu_engine::command_prelude::*; + use std::collections::BTreeMap; use std::{fmt, str}; use unicode_segmentation::UnicodeSegmentation; @@ -29,6 +30,10 @@ impl Command for SubCommand { vec!["count", "word", "character", "unicode", "wc"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -39,6 +44,15 @@ impl Command for SubCommand { stats(engine_state, call, input) } + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + stats(working_set.permanent(), call, input) + } + fn examples(&self) -> Vec { vec![ Example { @@ -108,7 +122,7 @@ fn stats( ), } }, - engine_state.ctrlc.clone(), + engine_state.signals(), ) } diff --git a/crates/nu-command/src/strings/str_/substring.rs b/crates/nu-command/src/strings/str_/substring.rs index 55871401ef..10464580c7 100644 --- a/crates/nu-command/src/strings/str_/substring.rs +++ b/crates/nu-command/src/strings/str_/substring.rs @@ -1,11 +1,10 @@ -use crate::grapheme_flags; +use crate::{grapheme_flags, grapheme_flags_const}; use nu_cmd_base::{ input_handler::{operate, CmdArgument}, util, }; use nu_engine::command_prelude::*; -use nu_protocol::Range; -use std::cmp::Ordering; +use nu_protocol::{engine::StateWorkingSet, Range}; use unicode_segmentation::UnicodeSegmentation; #[derive(Clone)] @@ -70,13 +69,17 @@ impl Command for SubCommand { } fn usage(&self) -> &str { - "Get part of a string. Note that the start is included but the end is excluded, and that the first character of a string is index 0." + "Get part of a string. Note that the first character of a string is index 0." } fn search_terms(&self) -> Vec<&str> { vec!["slice"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -100,7 +103,38 @@ impl Command for SubCommand { cell_paths, graphemes: grapheme_flags(engine_state, stack, call)?, }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + operate(action, args, input, call.head, engine_state.signals()) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let range: Range = call.req_const(working_set, 0)?; + + let indexes = match util::process_range(&range) { + Ok(idxs) => idxs.into(), + Err(processing_error) => { + return Err(processing_error("could not perform substring", call.head)) + } + }; + + let cell_paths: Vec = call.rest_const(working_set, 1)?; + let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); + let args = Arguments { + indexes, + cell_paths, + graphemes: grapheme_flags_const(working_set, call)?, + }; + operate( + action, + args, + input, + call.head, + working_set.permanent().signals(), + ) } fn examples(&self) -> Vec { @@ -108,14 +142,19 @@ impl Command for SubCommand { Example { description: "Get a substring \"nushell\" from the text \"good nushell\" using a range", - example: " 'good nushell' | str substring 5..12", + example: " 'good nushell' | str substring 5..11", result: Some(Value::test_string("nushell")), }, Example { description: "Count indexes and split using grapheme clusters", - example: " '🇯🇵ほげ ふが ぴよ' | str substring --grapheme-clusters 4..6", + example: " '🇯🇵ほげ ふが ぴよ' | str substring --grapheme-clusters 4..5", result: Some(Value::test_string("ふが")), }, + Example { + description: "sub string by negative index", + example: " 'good nushell' | str substring 5..-2", + result: Some(Value::test_string("nushel")), + }, ] } } @@ -132,56 +171,46 @@ fn action(input: &Value, args: &Arguments, head: Span) -> Value { options.0 }; let end: isize = if options.1 < 0 { - std::cmp::max(len + options.1, 0) + options.1 + len } else { options.1 }; - if start < len && end >= 0 { - match start.cmp(&end) { - Ordering::Equal => Value::string("", head), - Ordering::Greater => Value::error( - ShellError::TypeMismatch { - err_message: "End must be greater than or equal to Start".to_string(), - span: head, - }, - head, - ), - Ordering::Less => Value::string( - { - if end == isize::MAX { - if args.graphemes { - s.graphemes(true) - .skip(start as usize) - .collect::>() - .join("") - } else { - String::from_utf8_lossy( - &s.bytes().skip(start as usize).collect::>(), - ) - .to_string() - } - } else if args.graphemes { + if start > end { + Value::string("", head) + } else { + Value::string( + { + if end == isize::MAX { + if args.graphemes { s.graphemes(true) .skip(start as usize) - .take((end - start) as usize) .collect::>() .join("") } else { String::from_utf8_lossy( - &s.bytes() - .skip(start as usize) - .take((end - start) as usize) - .collect::>(), + &s.bytes().skip(start as usize).collect::>(), ) .to_string() } - }, - head, - ), - } - } else { - Value::string("", head) + } else if args.graphemes { + s.graphemes(true) + .skip(start as usize) + .take((end - start + 1) as usize) + .collect::>() + .join("") + } else { + String::from_utf8_lossy( + &s.bytes() + .skip(start as usize) + .take((end - start + 1) as usize) + .collect::>(), + ) + .to_string() + } + }, + head, + ) } } // Propagate errors by explicitly matching them before the final case. @@ -208,6 +237,7 @@ mod tests { test_examples(SubCommand {}) } + #[derive(Debug)] struct Expectation<'a> { options: (isize, isize), expected: &'a str, @@ -231,18 +261,19 @@ mod tests { let word = Value::test_string("andres"); let cases = vec![ - expectation("a", (0, 1)), - expectation("an", (0, 2)), - expectation("and", (0, 3)), - expectation("andr", (0, 4)), - expectation("andre", (0, 5)), + expectation("a", (0, 0)), + expectation("an", (0, 1)), + expectation("and", (0, 2)), + expectation("andr", (0, 3)), + expectation("andre", (0, 4)), + expectation("andres", (0, 5)), expectation("andres", (0, 6)), - expectation("", (0, -6)), - expectation("a", (0, -5)), - expectation("an", (0, -4)), - expectation("and", (0, -3)), - expectation("andr", (0, -2)), - expectation("andre", (0, -1)), + expectation("a", (0, -6)), + expectation("an", (0, -5)), + expectation("and", (0, -4)), + expectation("andr", (0, -3)), + expectation("andre", (0, -2)), + expectation("andres", (0, -1)), // str substring [ -4 , _ ] // str substring -4 , expectation("dres", (-4, isize::MAX)), @@ -257,6 +288,7 @@ mod tests { ]; for expectation in &cases { + println!("{:?}", expectation); let expected = expectation.expected; let actual = action( &word, diff --git a/crates/nu-command/src/strings/str_/trim/trim_.rs b/crates/nu-command/src/strings/str_/trim/trim_.rs index ee414d0da6..76d886a010 100644 --- a/crates/nu-command/src/strings/str_/trim/trim_.rs +++ b/crates/nu-command/src/strings/str_/trim/trim_.rs @@ -71,6 +71,10 @@ impl Command for SubCommand { vec!["whitespace", "strip", "lstrip", "rstrip"] } + fn is_const(&self) -> bool { + true + } + fn run( &self, engine_state: &EngineState, @@ -79,44 +83,37 @@ impl Command for SubCommand { input: PipelineData, ) -> Result { let character = call.get_flag::>(engine_state, stack, "char")?; - let to_trim = match character.as_ref() { - Some(v) => { - if v.item.chars().count() > 1 { - return Err(ShellError::GenericError { - error: "Trim only works with single character".into(), - msg: "needs single character".into(), - span: Some(v.span), - help: None, - inner: vec![], - }); - } - v.item.chars().next() - } - None => None, - }; let cell_paths: Vec = call.rest(engine_state, stack, 0)?; - let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); - let mode = match cell_paths { - None => ActionMode::Global, - Some(_) => ActionMode::Local, - }; - let left = call.has_flag(engine_state, stack, "left")?; let right = call.has_flag(engine_state, stack, "right")?; - let trim_side = match (left, right) { - (true, true) => TrimSide::Both, - (true, false) => TrimSide::Left, - (false, true) => TrimSide::Right, - (false, false) => TrimSide::Both, - }; - - let args = Arguments { - to_trim, - trim_side, + run( + character, cell_paths, - mode, - }; - operate(action, args, input, call.head, engine_state.ctrlc.clone()) + (left, right), + call, + input, + engine_state, + ) + } + + fn run_const( + &self, + working_set: &StateWorkingSet, + call: &Call, + input: PipelineData, + ) -> Result { + let character = call.get_flag_const::>(working_set, "char")?; + let cell_paths: Vec = call.rest_const(working_set, 0)?; + let left = call.has_flag_const(working_set, "left")?; + let right = call.has_flag_const(working_set, "right")?; + run( + character, + cell_paths, + (left, right), + call, + input, + working_set.permanent(), + ) } fn examples(&self) -> Vec { @@ -150,6 +147,52 @@ impl Command for SubCommand { } } +fn run( + character: Option>, + cell_paths: Vec, + (left, right): (bool, bool), + call: &Call, + input: PipelineData, + engine_state: &EngineState, +) -> Result { + let to_trim = match character.as_ref() { + Some(v) => { + if v.item.chars().count() > 1 { + return Err(ShellError::GenericError { + error: "Trim only works with single character".into(), + msg: "needs single character".into(), + span: Some(v.span), + help: None, + inner: vec![], + }); + } + v.item.chars().next() + } + None => None, + }; + + let cell_paths = (!cell_paths.is_empty()).then_some(cell_paths); + let mode = match cell_paths { + None => ActionMode::Global, + Some(_) => ActionMode::Local, + }; + + let trim_side = match (left, right) { + (true, true) => TrimSide::Both, + (true, false) => TrimSide::Left, + (false, true) => TrimSide::Right, + (false, false) => TrimSide::Both, + }; + + let args = Arguments { + to_trim, + trim_side, + cell_paths, + mode, + }; + operate(action, args, input, call.head, engine_state.signals()) +} + #[derive(Debug, Copy, Clone)] pub enum ActionMode { Local, diff --git a/crates/nu-command/src/system/exec.rs b/crates/nu-command/src/system/exec.rs index 21c98f121b..018017cfb2 100644 --- a/crates/nu-command/src/system/exec.rs +++ b/crates/nu-command/src/system/exec.rs @@ -1,7 +1,4 @@ -use super::run_external::create_external_command; -#[allow(deprecated)] -use nu_engine::{command_prelude::*, current_dir}; -use nu_protocol::OutDest; +use nu_engine::{command_prelude::*, env_to_strings}; #[derive(Clone)] pub struct Exec; @@ -35,7 +32,66 @@ On Windows based systems, Nushell will wait for the command to finish and then e call: &Call, _input: PipelineData, ) -> Result { - exec(engine_state, stack, call) + let cwd = engine_state.cwd(Some(stack))?; + + // Find the absolute path to the executable. If the command is not + // found, display a helpful error message. + let name: Spanned = call.req(engine_state, stack, 0)?; + let executable = { + let paths = nu_engine::env::path_str(engine_state, stack, call.head)?; + let Some(executable) = crate::which(&name.item, &paths, cwd.as_ref()) else { + return Err(crate::command_not_found( + &name.item, + call.head, + engine_state, + stack, + )); + }; + executable + }; + + // Create the command. + let mut command = std::process::Command::new(executable); + + // Configure PWD. + command.current_dir(cwd); + + // Configure environment variables. + let envs = env_to_strings(engine_state, stack)?; + command.env_clear(); + command.envs(envs); + + // Configure args. + let args = crate::eval_arguments_from_call(engine_state, stack, call)?; + command.args(args.into_iter().map(|s| s.item)); + + // Execute the child process, replacing/terminating the current process + // depending on platform. + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + + let err = command.exec(); + Err(ShellError::ExternalCommand { + label: "Failed to exec into new process".into(), + help: err.to_string(), + span: call.head, + }) + } + #[cfg(windows)] + { + let mut child = command.spawn().map_err(|err| ShellError::ExternalCommand { + label: "Failed to exec into new process".into(), + help: err.to_string(), + span: call.head, + })?; + let status = child.wait().map_err(|err| ShellError::ExternalCommand { + label: "Failed to wait for child process".into(), + help: err.to_string(), + span: call.head, + })?; + std::process::exit(status.code().expect("status.code() succeeds on Windows")) + } } fn examples(&self) -> Vec { @@ -53,57 +109,3 @@ On Windows based systems, Nushell will wait for the command to finish and then e ] } } - -fn exec( - engine_state: &EngineState, - stack: &mut Stack, - call: &Call, -) -> Result { - let mut external_command = create_external_command(engine_state, stack, call)?; - external_command.out = OutDest::Inherit; - external_command.err = OutDest::Inherit; - - #[allow(deprecated)] - let cwd = current_dir(engine_state, stack)?; - let mut command = external_command.spawn_simple_command(&cwd.to_string_lossy())?; - command.current_dir(cwd); - command.envs(external_command.env_vars); - - // this either replaces our process and should not return, - // or the exec fails and we get an error back - exec_impl(command, call.head) -} - -#[cfg(unix)] -fn exec_impl(mut command: std::process::Command, span: Span) -> Result { - use std::os::unix::process::CommandExt; - - let error = command.exec(); - - Err(ShellError::GenericError { - error: "Error on exec".into(), - msg: error.to_string(), - span: Some(span), - help: None, - inner: vec![], - }) -} - -#[cfg(windows)] -fn exec_impl(mut command: std::process::Command, span: Span) -> Result { - match command.spawn() { - Ok(mut child) => match child.wait() { - Ok(status) => std::process::exit(status.code().unwrap_or(0)), - Err(e) => Err(ShellError::ExternalCommand { - label: "Error in external command".into(), - help: e.to_string(), - span, - }), - }, - Err(e) => Err(ShellError::ExternalCommand { - label: "Error spawning external command".into(), - help: e.to_string(), - span, - }), - } -} diff --git a/crates/nu-command/src/system/mod.rs b/crates/nu-command/src/system/mod.rs index 788e1b7740..a87aac8a07 100644 --- a/crates/nu-command/src/system/mod.rs +++ b/crates/nu-command/src/system/mod.rs @@ -5,6 +5,8 @@ mod nu_check; target_os = "android", target_os = "linux", target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", target_os = "macos", target_os = "windows" ))] @@ -23,13 +25,15 @@ pub use nu_check::NuCheck; target_os = "android", target_os = "linux", target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", target_os = "macos", target_os = "windows" ))] pub use ps::Ps; #[cfg(windows)] pub use registry_query::RegistryQuery; -pub use run_external::{External, ExternalCommand}; +pub use run_external::{command_not_found, eval_arguments_from_call, which, External}; pub use sys::*; pub use uname::UName; pub use which_::Which; diff --git a/crates/nu-command/src/system/nu_check.rs b/crates/nu-command/src/system/nu_check.rs index f9e0879c00..334569c79e 100644 --- a/crates/nu-command/src/system/nu_check.rs +++ b/crates/nu-command/src/system/nu_check.rs @@ -87,7 +87,7 @@ impl Command for NuCheck { &path_str.item, engine_state, stack, - get_dirs_var_from_call(call), + get_dirs_var_from_call(stack, call), ) { Ok(path) => { if let Some(path) = path { diff --git a/crates/nu-command/src/system/ps.rs b/crates/nu-command/src/system/ps.rs index 922bf7915b..859575abbd 100644 --- a/crates/nu-command/src/system/ps.rs +++ b/crates/nu-command/src/system/ps.rs @@ -2,20 +2,13 @@ use itertools::Itertools; use nu_engine::command_prelude::*; -#[cfg(all( - unix, - not(target_os = "freebsd"), - not(target_os = "macos"), - not(target_os = "windows"), - not(target_os = "android"), -))] +#[cfg(target_os = "linux")] use procfs::WithCurrentSystemInfo; use std::time::Duration; #[derive(Clone)] pub struct Ps; -#[cfg(not(target_os = "freebsd"))] impl Command for Ps { fn name(&self) -> &str { "ps" @@ -82,7 +75,6 @@ impl Command for Ps { } } -#[cfg(not(target_os = "freebsd"))] fn run_ps( engine_state: &EngineState, stack: &mut Stack, @@ -111,12 +103,7 @@ fn run_ps( if long { record.push("command", Value::string(proc.command(), span)); - #[cfg(all( - unix, - not(target_os = "macos"), - not(target_os = "windows"), - not(target_os = "android"), - ))] + #[cfg(target_os = "linux")] { let proc_stat = proc .curr_proc @@ -195,5 +182,5 @@ fn run_ps( Ok(output .into_iter() - .into_pipeline_data(span, engine_state.ctrlc.clone())) + .into_pipeline_data(span, engine_state.signals().clone())) } diff --git a/crates/nu-command/src/system/registry_query.rs b/crates/nu-command/src/system/registry_query.rs index 1e2a328356..9b68059ffa 100644 --- a/crates/nu-command/src/system/registry_query.rs +++ b/crates/nu-command/src/system/registry_query.rs @@ -106,7 +106,7 @@ fn registry_query( *registry_key_span, )) } - Ok(reg_values.into_pipeline_data(call_span, engine_state.ctrlc.clone())) + Ok(reg_values.into_pipeline_data(call_span, engine_state.signals().clone())) } else { match registry_value { Some(value) => { diff --git a/crates/nu-command/src/system/run_external.rs b/crates/nu-command/src/system/run_external.rs index b12b89263c..a5cf343970 100644 --- a/crates/nu-command/src/system/run_external.rs +++ b/crates/nu-command/src/system/run_external.rs @@ -1,15 +1,16 @@ use nu_cmd_base::hook::eval_hook; use nu_engine::{command_prelude::*, env_to_strings, get_eval_expression}; -use nu_protocol::{ast::Expr, did_you_mean, process::ChildProcess, ByteStream, NuGlob, OutDest}; +use nu_path::{dots::expand_ndots, expand_tilde}; +use nu_protocol::{did_you_mean, process::ChildProcess, ByteStream, NuGlob, OutDest, Signals}; use nu_system::ForegroundChild; use nu_utils::IgnoreCaseExt; -use os_pipe::PipeReader; use pathdiff::diff_paths; use std::{ - collections::HashMap, + borrow::Cow, + ffi::{OsStr, OsString}, io::Write, path::{Path, PathBuf}, - process::{Command as CommandSys, Stdio}, + process::Stdio, sync::Arc, thread, }; @@ -29,8 +30,16 @@ impl Command for External { fn signature(&self) -> nu_protocol::Signature { Signature::build(self.name()) .input_output_types(vec![(Type::Any, Type::Any)]) - .required("command", SyntaxShape::String, "External command to run.") - .rest("args", SyntaxShape::Any, "Arguments for external command.") + .required( + "command", + SyntaxShape::OneOf(vec![SyntaxShape::GlobPattern, SyntaxShape::String]), + "External command to run.", + ) + .rest( + "args", + SyntaxShape::OneOf(vec![SyntaxShape::GlobPattern, SyntaxShape::Any]), + "Arguments for external command.", + ) .category(Category::System) } @@ -41,8 +50,146 @@ impl Command for External { call: &Call, input: PipelineData, ) -> Result { - let command = create_external_command(engine_state, stack, call)?; - command.run_with_input(engine_state, stack, input, false) + let cwd = engine_state.cwd(Some(stack))?; + + let name: Value = call.req(engine_state, stack, 0)?; + + let name_str: Cow = match &name { + Value::Glob { val, .. } => Cow::Borrowed(val), + Value::String { val, .. } => Cow::Borrowed(val), + _ => Cow::Owned(name.clone().coerce_into_string()?), + }; + + let expanded_name = match &name { + // Expand tilde and ndots on the name if it's a bare string / glob (#13000) + Value::Glob { no_expand, .. } if !*no_expand => { + expand_ndots_safe(expand_tilde(&*name_str)) + } + _ => Path::new(&*name_str).to_owned(), + }; + + // Find the absolute path to the executable. On Windows, set the + // executable to "cmd.exe" if it's is a CMD internal command. If the + // command is not found, display a helpful error message. + let executable = if cfg!(windows) && is_cmd_internal_command(&name_str) { + PathBuf::from("cmd.exe") + } else { + // Determine the PATH to be used and then use `which` to find it - though this has no + // effect if it's an absolute path already + let paths = nu_engine::env::path_str(engine_state, stack, call.head)?; + let Some(executable) = which(expanded_name, &paths, cwd.as_ref()) else { + return Err(command_not_found(&name_str, call.head, engine_state, stack)); + }; + executable + }; + + // Create the command. + let mut command = std::process::Command::new(executable); + + // Configure PWD. + command.current_dir(cwd); + + // Configure environment variables. + let envs = env_to_strings(engine_state, stack)?; + command.env_clear(); + command.envs(envs); + + // Configure args. + let args = eval_arguments_from_call(engine_state, stack, call)?; + #[cfg(windows)] + if is_cmd_internal_command(&name_str) { + use std::os::windows::process::CommandExt; + + // The /D flag disables execution of AutoRun commands from registry. + // The /C flag followed by a command name instructs CMD to execute + // that command and quit. + command.args(["/D", "/C", &name_str]); + for arg in &args { + command.raw_arg(escape_cmd_argument(arg)?); + } + } else { + command.args(args.into_iter().map(|s| s.item)); + } + #[cfg(not(windows))] + command.args(args.into_iter().map(|s| s.item)); + + // Configure stdout and stderr. If both are set to `OutDest::Pipe`, + // we'll setup a pipe that merge two streams into one. + let stdout = stack.stdout(); + let stderr = stack.stderr(); + let merged_stream = if matches!(stdout, OutDest::Pipe) && matches!(stderr, OutDest::Pipe) { + let (reader, writer) = os_pipe::pipe()?; + command.stdout(writer.try_clone()?); + command.stderr(writer); + Some(reader) + } else { + command.stdout(Stdio::try_from(stdout)?); + command.stderr(Stdio::try_from(stderr)?); + None + }; + + // Configure stdin. We'll try connecting input to the child process + // directly. If that's not possible, we'll setup a pipe and spawn a + // thread to copy data into the child process. + let data_to_copy_into_stdin = match input { + PipelineData::ByteStream(stream, metadata) => match stream.into_stdio() { + Ok(stdin) => { + command.stdin(stdin); + None + } + Err(stream) => { + command.stdin(Stdio::piped()); + Some(PipelineData::ByteStream(stream, metadata)) + } + }, + PipelineData::Empty => { + command.stdin(Stdio::inherit()); + None + } + value => { + command.stdin(Stdio::piped()); + Some(value) + } + }; + + // Log the command we're about to run in case it's useful for debugging purposes. + log::trace!("run-external spawning: {command:?}"); + + // Spawn the child process. On Unix, also put the child process to + // foreground if we're in an interactive session. + #[cfg(windows)] + let mut child = ForegroundChild::spawn(command)?; + #[cfg(unix)] + let mut child = ForegroundChild::spawn( + command, + engine_state.is_interactive, + &engine_state.pipeline_externals_state, + )?; + + // If we need to copy data into the child process, do it now. + if let Some(data) = data_to_copy_into_stdin { + let stdin = child.as_mut().stdin.take().expect("stdin is piped"); + let engine_state = engine_state.clone(); + let stack = stack.clone(); + thread::Builder::new() + .name("external stdin worker".into()) + .spawn(move || { + let _ = write_pipeline_data(engine_state, stack, data, stdin); + }) + .err_span(call.head)?; + } + + // Wrap the output into a `PipelineData::ByteStream`. + let child = ChildProcess::new( + child, + merged_stream, + matches!(stderr, OutDest::Pipe), + call.head, + )?; + Ok(PipelineData::ByteStream( + ByteStream::child(child, call.head), + None, + )) } fn examples(&self) -> Vec { @@ -66,678 +213,436 @@ impl Command for External { } } -/// Creates ExternalCommand from a call -pub fn create_external_command( +/// Evaluate all arguments from a call, performing expansions when necessary. +pub fn eval_arguments_from_call( engine_state: &EngineState, stack: &mut Stack, call: &Call, -) -> Result { - let name: Spanned = call.req(engine_state, stack, 0)?; - - // Translate environment variables from Values to Strings - let env_vars_str = env_to_strings(engine_state, stack)?; - - fn value_as_spanned(value: Value) -> Result, ShellError> { - let span = value.span(); - - value - .coerce_string() - .map(|item| Spanned { item, span }) - .map_err(|_| ShellError::ExternalCommand { - label: format!("Cannot convert {} to a string", value.get_type()), - help: "All arguments to an external command need to be string-compatible".into(), - span, - }) - } - +) -> Result>, ShellError> { + let cwd = engine_state.cwd(Some(stack))?; let eval_expression = get_eval_expression(engine_state); + let call_args = call.rest_iter_flattened(engine_state, stack, eval_expression, 1)?; + let mut args: Vec> = Vec::with_capacity(call_args.len()); - let mut spanned_args = vec![]; - let mut arg_keep_raw = vec![]; - for (arg, spread) in call.rest_iter(1) { - match eval_expression(engine_state, stack, arg)? { - Value::List { vals, .. } => { - if spread { - // turn all the strings in the array into params. - // Example: one_arg may be something like ["ls" "-a"] - // convert it to "ls" "-a" - for v in vals { - spanned_args.push(value_as_spanned(v)?); - // for arguments in list, it's always treated as a whole arguments - arg_keep_raw.push(true); - } - } else { - return Err(ShellError::CannotPassListToExternal { - arg: String::from_utf8_lossy(engine_state.get_span_contents(arg.span)) - .into(), - span: arg.span, - }); - } - } - val => { - if spread { - return Err(ShellError::CannotSpreadAsList { span: arg.span }); - } else { - spanned_args.push(value_as_spanned(val)?); - match arg.expr { - // refer to `parse_dollar_expr` function - // the expression type of $variable_name, $"($variable_name)" - // will be Expr::StringInterpolation, Expr::FullCellPath - Expr::StringInterpolation(_) | Expr::FullCellPath(_) => { - arg_keep_raw.push(true) - } - _ => arg_keep_raw.push(false), - } - } - } + for arg in call_args { + let span = arg.span(); + match arg { + // Expand globs passed to run-external + Value::Glob { val, no_expand, .. } if !no_expand => args.extend( + expand_glob(&val, cwd.as_std_path(), span, engine_state.signals())? + .into_iter() + .map(|s| s.into_spanned(span)), + ), + other => args + .push(OsString::from(coerce_into_string(engine_state, other)?).into_spanned(span)), } } - - Ok(ExternalCommand { - name, - args: spanned_args, - arg_keep_raw, - out: stack.stdout().clone(), - err: stack.stderr().clone(), - env_vars: env_vars_str, - }) + Ok(args) } -#[derive(Clone)] -pub struct ExternalCommand { - pub name: Spanned, - pub args: Vec>, - pub arg_keep_raw: Vec, - pub out: OutDest, - pub err: OutDest, - pub env_vars: HashMap, +/// Custom `coerce_into_string()`, including globs, since those are often args to `run-external` +/// as well +fn coerce_into_string(engine_state: &EngineState, val: Value) -> Result { + match val { + Value::List { .. } => Err(ShellError::CannotPassListToExternal { + arg: String::from_utf8_lossy(engine_state.get_span_contents(val.span())).into_owned(), + span: val.span(), + }), + Value::Glob { val, .. } => Ok(val), + _ => val.coerce_into_string(), + } } -impl ExternalCommand { - pub fn run_with_input( - &self, - engine_state: &EngineState, - stack: &mut Stack, - input: PipelineData, - reconfirm_command_name: bool, - ) -> Result { - let head = self.name.span; +/// Performs glob expansion on `arg`. If the expansion found no matches or the pattern +/// is not a valid glob, then this returns the original string as the expansion result. +/// +/// Note: This matches the default behavior of Bash, but is known to be +/// error-prone. We might want to change this behavior in the future. +fn expand_glob( + arg: &str, + cwd: &Path, + span: Span, + signals: &Signals, +) -> Result, ShellError> { + const GLOB_CHARS: &[char] = &['*', '?', '[']; - #[cfg(windows)] - let (child, reader, input) = { - // We may need to run `create_process` again, so we have to clone the underlying - // file or pipe in `input` here first. - let (input_consumed, stdin) = match &input { - PipelineData::ByteStream(stream, ..) => match stream.source() { - nu_protocol::ByteStreamSource::Read(_) => (false, Stdio::piped()), - nu_protocol::ByteStreamSource::File(file) => { - (true, file.try_clone().err_span(head)?.into()) - } - nu_protocol::ByteStreamSource::Child(child) => { - if let Some(nu_protocol::process::ChildPipe::Pipe(pipe)) = &child.stdout { - (true, pipe.try_clone().err_span(head)?.into()) - } else { - (false, Stdio::piped()) - } - } - }, - PipelineData::Empty => (false, Stdio::inherit()), - _ => (false, Stdio::piped()), - }; - - let mut input = input; - let (cmd, mut reader) = self.create_process(stdin, false, head)?; - let child = match ForegroundChild::spawn(cmd) { - Ok(child) => { - if input_consumed { - input = PipelineData::Empty; - } - Ok(child) - } - Err(err) => { - // Running external commands on Windows has 2 points of complication: - // 1. Some common Windows commands are actually built in to cmd.exe, not executables in their own right. - // 2. We need to let users run batch scripts etc. (.bat, .cmd) without typing their extension - - // To support these situations, we have a fallback path that gets run if a command - // fails to be run as a normal executable: - // 1. "shell out" to cmd.exe if the command is a known cmd.exe internal command - // 2. Otherwise, use `which-rs` to look for batch files etc. then run those in cmd.exe - - // set the default value, maybe we'll override it later - let mut child = Err(err); - - // This has the full list of cmd.exe "internal" commands: https://ss64.com/nt/syntax-internal.html - // I (Reilly) went through the full list and whittled it down to ones that are potentially useful: - const CMD_INTERNAL_COMMANDS: [&str; 9] = [ - "ASSOC", "CLS", "ECHO", "FTYPE", "MKLINK", "PAUSE", "START", "VER", "VOL", - ]; - let command_name = &self.name.item; - let looks_like_cmd_internal = CMD_INTERNAL_COMMANDS - .iter() - .any(|&cmd| command_name.eq_ignore_ascii_case(cmd)); - - let (data, stdin) = extract_stdio(input); - input = data; - - if looks_like_cmd_internal { - let (cmd, new_reader) = self.create_process(stdin, true, head)?; - reader = new_reader; - child = ForegroundChild::spawn(cmd); - } else { - #[cfg(feature = "which-support")] - { - // maybe it's a batch file (foo.cmd) and the user typed `foo`. Try to find it with `which-rs` - // TODO: clean this up with an if-let chain once those are stable - if let Ok(path) = - nu_engine::env::path_str(engine_state, stack, self.name.span) - { - if let Some(cwd) = self.env_vars.get("PWD") { - // append cwd to PATH so `which-rs` looks in the cwd too. - // this approximates what cmd.exe does. - let path_with_cwd = format!("{};{}", cwd, path); - if let Ok(which_path) = - which::which_in(&self.name.item, Some(path_with_cwd), cwd) - { - if let Some(file_name) = which_path.file_name() { - if !file_name - .to_string_lossy() - .eq_ignore_case(command_name) - { - // which-rs found an executable file with a slightly different name - // than the one the user tried. Let's try running it - let mut new_command = self.clone(); - new_command.name = Spanned { - item: file_name.to_string_lossy().to_string(), - span: self.name.span, - }; - let (cmd, new_reader) = new_command - .create_process(stdin, true, head)?; - reader = new_reader; - child = ForegroundChild::spawn(cmd); - } - } - } - } - } - } - } - - child - } - }; - - (child, reader, input) - }; - - #[cfg(unix)] - let (child, reader, input) = { - let (input, stdin) = extract_stdio(input); - let (cmd, reader) = self.create_process(stdin, false, head)?; - let child = ForegroundChild::spawn( - cmd, - engine_state.is_interactive, - &engine_state.pipeline_externals_state, - ); - (child, reader, input) - }; - - match child { - Err(err) => { - match err.kind() { - // If file not found, try suggesting alternative commands to the user - std::io::ErrorKind::NotFound => { - // recommend a replacement if the user tried a removed command - let command_name_lower = self.name.item.to_lowercase(); - let removed_from_nu = crate::removed_commands(); - if removed_from_nu.contains_key(&command_name_lower) { - let replacement = match removed_from_nu.get(&command_name_lower) { - Some(s) => s.clone(), - None => "".to_string(), - }; - return Err(ShellError::RemovedCommand { - removed: command_name_lower, - replacement, - span: self.name.span, - }); - } - - let suggestion = suggest_command(&self.name.item, engine_state); - let label = match suggestion { - Some(s) => { - if reconfirm_command_name { - format!( - "'{}' was not found; did you mean '{s}'?", - self.name.item - ) - } else { - let cmd_name = &self.name.item; - let maybe_module = engine_state - .which_module_has_decl(cmd_name.as_bytes(), &[]); - if let Some(module_name) = maybe_module { - let module_name = String::from_utf8_lossy(module_name); - let new_name = &[module_name.as_ref(), cmd_name].join(" "); - - if engine_state - .find_decl(new_name.as_bytes(), &[]) - .is_some() - { - format!("command '{cmd_name}' was not found but it was imported from module '{module_name}'; try using `{new_name}`") - } else { - format!("command '{cmd_name}' was not found but it exists in module '{module_name}'; try importing it with `use`") - } - } else { - // If command and suggestion are the same, display not found - if cmd_name == &s { - format!("'{cmd_name}' was not found") - } else { - format!("did you mean '{s}'?") - } - } - } - } - None => { - if reconfirm_command_name { - format!("executable '{}' was not found", self.name.item) - } else { - "executable was not found".into() - } - } - }; - - let mut err_str = err.to_string(); - if engine_state.is_interactive { - let mut engine_state = engine_state.clone(); - if let Some(hook) = engine_state.config.hooks.command_not_found.clone() - { - let canary = "ENTERED_COMMAND_NOT_FOUND"; - let stack = &mut stack.start_capture(); - if stack.has_env_var(&engine_state, canary) { - return Err(ShellError::ExternalCommand { - label: "command_not_found handler could not be run".into(), - help: "make sure the command_not_found closure itself does not use unknown commands".to_string(), - span: self.name.span, - }); - } - stack.add_env_var( - canary.to_string(), - Value::bool(true, Span::unknown()), - ); - match eval_hook( - &mut engine_state, - stack, - None, - vec![( - "cmd_name".into(), - Value::string(self.name.item.to_string(), self.name.span), - )], - &hook, - "command_not_found", - ) { - Ok(PipelineData::Value(Value::String { val, .. }, ..)) => { - err_str = format!("{}\n{}", err_str, val); - } - - Err(err) => { - stack.remove_env_var(&engine_state, canary); - return Err(err); - } - _ => {} - } - stack.remove_env_var(&engine_state, canary); - } - } - - Err(ShellError::ExternalCommand { - label, - help: err_str, - span: self.name.span, - }) - } - // otherwise, a default error message - _ => Err(ShellError::ExternalCommand { - label: "can't run executable".into(), - help: err.to_string(), - span: self.name.span, - }), - } - } - Ok(mut child) => { - if !input.is_nothing() { - let mut engine_state = engine_state.clone(); - let mut stack = stack.clone(); - - // Turn off color as we pass data through - Arc::make_mut(&mut engine_state.config).use_ansi_coloring = false; - - // Pipe input into the external command's stdin - if let Some(mut stdin_write) = child.as_mut().stdin.take() { - thread::Builder::new() - .name("external stdin worker".to_string()) - .spawn(move || { - let input = match input { - input @ PipelineData::ByteStream(..) => input, - input @ PipelineData::Value(Value::Binary { .. }, ..) => input, - input => { - let stack = &mut stack.start_capture(); - // Attempt to render the input as a table before piping it to the external. - // This is important for pagers like `less`; - // they need to get Nu data rendered for display to users. - // - // TODO: should we do something different for list inputs? - // Users often expect those to be piped to *nix tools as raw strings separated by newlines - crate::Table.run( - &engine_state, - stack, - &Call::new(head), - input, - )? - } - }; - - if let PipelineData::ByteStream(stream, ..) = input { - stream.write_to(&mut stdin_write)?; - } else { - for value in input.into_iter() { - let buf = value.coerce_into_binary()?; - stdin_write.write_all(&buf)?; - } - } - - Ok::<_, ShellError>(()) - }) - .err_span(head)?; - } - } - - let child = - ChildProcess::new(child, reader, matches!(self.err, OutDest::Pipe), head)?; - - Ok(PipelineData::ByteStream( - ByteStream::child(child, head), - None, - )) - } - } + // For an argument that doesn't include the GLOB_CHARS, just do the `expand_tilde` + // and `expand_ndots` expansion + if !arg.contains(GLOB_CHARS) { + let path = expand_ndots_safe(expand_tilde(arg)); + return Ok(vec![path.into()]); } - pub fn create_process( - &self, - stdin: Stdio, - use_cmd: bool, - span: Span, - ) -> Result<(CommandSys, Option), ShellError> { - let mut process = if let Some(d) = self.env_vars.get("PWD") { - let mut process = if use_cmd { - self.spawn_cmd_command(d) + // We must use `nu_engine::glob_from` here, in order to ensure we get paths from the correct + // dir + let glob = NuGlob::Expand(arg.to_owned()).into_spanned(span); + if let Ok((prefix, matches)) = nu_engine::glob_from(&glob, cwd, span, None) { + let mut result: Vec = vec![]; + + for m in matches { + signals.check(span)?; + if let Ok(arg) = m { + let arg = resolve_globbed_path_to_cwd_relative(arg, prefix.as_ref(), cwd); + result.push(arg.into()); } else { - self.create_command(d)? - }; - - // do not try to set current directory if cwd does not exist - if Path::new(&d).exists() { - process.current_dir(d); + result.push(arg.into()); } - process - } else { - return Err(ShellError::GenericError{ - error: "Current directory not found".into(), - msg: "did not find PWD environment variable".into(), - span: Some(span), - help: Some(concat!( - "The environment variable 'PWD' was not found. ", - "It is required to define the current directory when running an external command." - ).into()), - inner:Vec::new(), - }); - }; - - process.envs(&self.env_vars); - - // If the external is not the last command, its output will get piped - // either as a string or binary - let reader = if matches!(self.out, OutDest::Pipe) && matches!(self.err, OutDest::Pipe) { - let (reader, writer) = os_pipe::pipe()?; - let writer_clone = writer.try_clone()?; - process.stdout(writer); - process.stderr(writer_clone); - Some(reader) - } else { - process.stdout(Stdio::try_from(&self.out)?); - process.stderr(Stdio::try_from(&self.err)?); - None - }; - - process.stdin(stdin); - - Ok((process, reader)) - } - - fn create_command(&self, cwd: &str) -> Result { - // in all the other cases shell out - if cfg!(windows) { - //TODO. This should be modifiable from the config file. - // We could give the option to call from powershell - // for minimal builds cwd is unused - if self.name.item.ends_with(".cmd") || self.name.item.ends_with(".bat") { - Ok(self.spawn_cmd_command(cwd)) - } else { - self.spawn_simple_command(cwd) - } - } else { - self.spawn_simple_command(cwd) - } - } - - /// Spawn a command without shelling out to an external shell - /// - /// Note that this function will not set the cwd or environment variables. - /// It only creates the command and adds arguments. - pub fn spawn_simple_command(&self, cwd: &str) -> Result { - let (head, _, _) = trim_enclosing_quotes(&self.name.item); - let head = nu_path::expand_to_real_path(head) - .to_string_lossy() - .to_string(); - - let mut process = std::process::Command::new(head); - process.env_clear(); - - for (arg, arg_keep_raw) in self.args.iter().zip(self.arg_keep_raw.iter()) { - trim_expand_and_apply_arg(&mut process, arg, arg_keep_raw, cwd); } - Ok(process) - } - - /// Spawn a cmd command with `cmd /c args...` - pub fn spawn_cmd_command(&self, cwd: &str) -> std::process::Command { - let mut process = std::process::Command::new("cmd"); - - // Disable AutoRun - // TODO: There should be a config option to enable/disable this - // Alternatively (even better) a config option to specify all the arguments to pass to cmd - process.arg("/D"); - - process.arg("/c"); - process.arg(&self.name.item); - for (arg, arg_keep_raw) in self.args.iter().zip(self.arg_keep_raw.iter()) { - // https://stackoverflow.com/questions/1200235/how-to-pass-a-quoted-pipe-character-to-cmd-exe - // cmd.exe needs to have a caret to escape a pipe - let arg = Spanned { - item: arg.item.replace('|', "^|"), - span: arg.span, - }; - - trim_expand_and_apply_arg(&mut process, &arg, arg_keep_raw, cwd) + // FIXME: do we want to special-case this further? We might accidentally expand when they don't + // intend to + if result.is_empty() { + result.push(arg.into()); } - process + Ok(result) + } else { + Ok(vec![arg.into()]) } } -fn trim_expand_and_apply_arg( - process: &mut CommandSys, - arg: &Spanned, - arg_keep_raw: &bool, - cwd: &str, -) { - // if arg is quoted, like "aa", 'aa', `aa`, or: - // if arg is a variable or String interpolation, like: $variable_name, $"($variable_name)" - // `as_a_whole` will be true, so nu won't remove the inner quotes. - let (trimmed_args, mut run_glob_expansion, mut keep_raw) = trim_enclosing_quotes(&arg.item); - if *arg_keep_raw { - keep_raw = true; - // it's a list or a variable, don't run glob expansion either - run_glob_expansion = false; - } - let mut arg = Spanned { - item: if keep_raw { - trimmed_args +fn resolve_globbed_path_to_cwd_relative( + path: PathBuf, + prefix: Option<&PathBuf>, + cwd: &Path, +) -> PathBuf { + if let Some(prefix) = prefix { + if let Ok(remainder) = path.strip_prefix(prefix) { + let new_prefix = if let Some(pfx) = diff_paths(prefix, cwd) { + pfx + } else { + prefix.to_path_buf() + }; + new_prefix.join(remainder) } else { - remove_quotes(trimmed_args) - }, - span: arg.span, - }; - if !keep_raw { - arg.item = nu_path::expand_tilde(arg.item) - .to_string_lossy() - .to_string(); - } - let cwd = PathBuf::from(cwd); - if arg.item.contains('*') && run_glob_expansion { - // we need to run glob expansion, so it's NeedExpand. - let path = Spanned { - item: NuGlob::Expand(arg.item.clone()), - span: arg.span, - }; - if let Ok((prefix, matches)) = nu_engine::glob_from(&path, &cwd, arg.span, None) { - let matches: Vec<_> = matches.collect(); - - // FIXME: do we want to special-case this further? We might accidentally expand when they don't - // intend to - if matches.is_empty() { - process.arg(&arg.item); - } - for m in matches { - if let Ok(arg) = m { - let arg = if let Some(prefix) = &prefix { - if let Ok(remainder) = arg.strip_prefix(prefix) { - let new_prefix = if let Some(pfx) = diff_paths(prefix, &cwd) { - pfx - } else { - prefix.to_path_buf() - }; - - new_prefix.join(remainder).to_string_lossy().to_string() - } else { - arg.to_string_lossy().to_string() - } - } else { - arg.to_string_lossy().to_string() - }; - - process.arg(&arg); - } else { - process.arg(&arg.item); - } - } + path } } else { - process.arg(&arg.item); + path } } -/// Given an invalid command name, try to suggest an alternative -fn suggest_command(attempted_command: &str, engine_state: &EngineState) -> Option { - let commands = engine_state.get_signatures(false); - let command_folded_case = attempted_command.to_folded_case(); - let search_term_match = commands.iter().find(|sig| { - sig.search_terms - .iter() - .any(|term| term.to_folded_case() == command_folded_case) - }); - match search_term_match { - Some(sig) => Some(sig.name.clone()), - None => { - let command_names: Vec = commands.iter().map(|sig| sig.name.clone()).collect(); - did_you_mean(&command_names, attempted_command) +/// Write `PipelineData` into `writer`. If `PipelineData` is not binary, it is +/// first rendered using the `table` command. +/// +/// Note: Avoid using this function when piping data from an external command to +/// another external command, because it copies data unnecessarily. Instead, +/// extract the pipe from the `PipelineData::ByteStream` of the first command +/// and hand it to the second command directly. +fn write_pipeline_data( + mut engine_state: EngineState, + mut stack: Stack, + data: PipelineData, + mut writer: impl Write, +) -> Result<(), ShellError> { + if let PipelineData::ByteStream(stream, ..) = data { + stream.write_to(writer)?; + } else if let PipelineData::Value(Value::Binary { val, .. }, ..) = data { + writer.write_all(&val)?; + } else { + stack.start_capture(); + + // Turn off color as we pass data through + Arc::make_mut(&mut engine_state.config).use_ansi_coloring = false; + + // Invoke the `table` command. + let output = + crate::Table.run(&engine_state, &mut stack, &Call::new(Span::unknown()), data)?; + + // Write the output. + for value in output { + let bytes = value.coerce_into_binary()?; + writer.write_all(&bytes)?; } } + Ok(()) } -/// This function returns a tuple with 3 items: -/// 1st item: trimmed string. -/// 2nd item: a boolean value indicate if it's ok to run glob expansion. -/// 3rd item: a boolean value indicate if we need to keep raw string. -fn trim_enclosing_quotes(input: &str) -> (String, bool, bool) { - let mut chars = input.chars(); +/// Returns a helpful error message given an invalid command name, +pub fn command_not_found( + name: &str, + span: Span, + engine_state: &EngineState, + stack: &mut Stack, +) -> ShellError { + // Run the `command_not_found` hook if there is one. + if let Some(hook) = &engine_state.config.hooks.command_not_found { + let mut stack = stack.start_capture(); + // Set a special environment variable to avoid infinite loops when the + // `command_not_found` hook triggers itself. + let canary = "ENTERED_COMMAND_NOT_FOUND"; + if stack.has_env_var(engine_state, canary) { + return ShellError::ExternalCommand { + label: format!( + "Command {name} not found while running the `command_not_found` hook" + ), + help: "Make sure the `command_not_found` hook itself does not use unknown commands" + .into(), + span, + }; + } + stack.add_env_var(canary.into(), Value::bool(true, Span::unknown())); - match (chars.next(), chars.next_back()) { - (Some('"'), Some('"')) => (chars.collect(), false, true), - (Some('\''), Some('\'')) => (chars.collect(), false, true), - // We treat back-quoted strings as bare words, so there's no need to keep them as raw strings - (Some('`'), Some('`')) => (chars.collect(), true, false), - _ => (input.to_string(), true, false), + let output = eval_hook( + &mut engine_state.clone(), + &mut stack, + None, + vec![("cmd_name".into(), Value::string(name, span))], + hook, + "command_not_found", + ); + + // Remove the special environment variable that we just set. + stack.remove_env_var(engine_state, canary); + + match output { + Ok(PipelineData::Value(Value::String { val, .. }, ..)) => { + return ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: val, + span, + }; + } + Err(err) => { + return err; + } + _ => { + // The hook did not return a string, so ignore it. + } + } + } + + // If the name is one of the removed commands, recommend a replacement. + if let Some(replacement) = crate::removed_commands().get(&name.to_lowercase()) { + return ShellError::RemovedCommand { + removed: name.to_lowercase(), + replacement: replacement.clone(), + span, + }; + } + + // The command might be from another module. Try to find it. + if let Some(module) = engine_state.which_module_has_decl(name.as_bytes(), &[]) { + let module = String::from_utf8_lossy(module); + // Is the command already imported? + let full_name = format!("{module} {name}"); + if engine_state.find_decl(full_name.as_bytes(), &[]).is_some() { + return ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: format!("Did you mean `{full_name}`?"), + span, + }; + } else { + return ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: format!("A command with that name exists in module `{module}`. Try importing it with `use`"), + span, + }; + } + } + + // Try to match the name with the search terms of existing commands. + let signatures = engine_state.get_signatures(false); + if let Some(sig) = signatures.iter().find(|sig| { + sig.search_terms + .iter() + .any(|term| term.to_folded_case() == name.to_folded_case()) + }) { + return ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: format!("Did you mean `{}`?", sig.name), + span, + }; + } + + // Try a fuzzy search on the names of all existing commands. + if let Some(cmd) = did_you_mean(signatures.iter().map(|sig| &sig.name), name) { + // The user is invoking an external command with the same name as a + // built-in command. Remind them of this. + if cmd == name { + return ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: "There is a built-in command with the same name".into(), + span, + }; + } + return ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: format!("Did you mean `{cmd}`?"), + span, + }; + } + + // We found nothing useful. Give up and return a generic error message. + ShellError::ExternalCommand { + label: format!("Command `{name}` not found"), + help: format!("`{name}` is neither a Nushell built-in or a known external command"), + span, } } -fn remove_quotes(input: String) -> String { - let mut chars = input.chars(); +/// Searches for the absolute path of an executable by name. `.bat` and `.cmd` +/// files are recognized as executables on Windows. +/// +/// This is a wrapper around `which::which_in()` except that, on Windows, it +/// also searches the current directory before any PATH entries. +/// +/// Note: the `which.rs` crate always uses PATHEXT from the environment. As +/// such, changing PATHEXT within Nushell doesn't work without updating the +/// actual environment of the Nushell process. +pub fn which(name: impl AsRef, paths: &str, cwd: &Path) -> Option { + #[cfg(windows)] + let paths = format!("{};{}", cwd.display(), paths); + which::which_in(name, Some(paths), cwd).ok() +} - match (chars.next_back(), input.contains('=')) { - (Some('"'), true) => chars - .collect::() - .replacen('"', "", 1) - .replace(r#"\""#, "\""), - (Some('\''), true) => chars.collect::().replacen('\'', "", 1), - _ => input, +/// Returns true if `name` is a (somewhat useful) CMD internal command. The full +/// list can be found at https://ss64.com/nt/syntax-internal.html +fn is_cmd_internal_command(name: &str) -> bool { + const COMMANDS: &[&str] = &[ + "ASSOC", "CLS", "ECHO", "FTYPE", "MKLINK", "PAUSE", "START", "VER", "VOL", + ]; + COMMANDS.iter().any(|cmd| cmd.eq_ignore_ascii_case(name)) +} + +/// Returns true if a string contains CMD special characters. +fn has_cmd_special_character(s: impl AsRef<[u8]>) -> bool { + s.as_ref() + .iter() + .any(|b| matches!(b, b'<' | b'>' | b'&' | b'|' | b'^')) +} + +/// Escape an argument for CMD internal commands. The result can be safely passed to `raw_arg()`. +#[cfg_attr(not(windows), allow(dead_code))] +fn escape_cmd_argument(arg: &Spanned) -> Result, ShellError> { + let Spanned { item: arg, span } = arg; + let bytes = arg.as_encoded_bytes(); + if bytes.iter().any(|b| matches!(b, b'\r' | b'\n' | b'%')) { + // \r and \n trunacte the rest of the arguments and % can expand environment variables + Err(ShellError::ExternalCommand { + label: + "Arguments to CMD internal commands cannot contain new lines or percent signs '%'" + .into(), + help: "some characters currently cannot be securely escaped".into(), + span: *span, + }) + } else if bytes.contains(&b'"') { + // If `arg` is already quoted by double quotes, confirm there's no + // embedded double quotes, then leave it as is. + if bytes.iter().filter(|b| **b == b'"').count() == 2 + && bytes.starts_with(b"\"") + && bytes.ends_with(b"\"") + { + Ok(Cow::Borrowed(arg)) + } else { + Err(ShellError::ExternalCommand { + label: "Arguments to CMD internal commands cannot contain embedded double quotes" + .into(), + help: "this case currently cannot be securely handled".into(), + span: *span, + }) + } + } else if bytes.contains(&b' ') || has_cmd_special_character(bytes) { + // If `arg` contains space or special characters, quote the entire argument by double quotes. + let mut new_str = OsString::new(); + new_str.push("\""); + new_str.push(arg); + new_str.push("\""); + Ok(Cow::Owned(new_str)) + } else { + // FIXME?: what if `arg.is_empty()`? + Ok(Cow::Borrowed(arg)) } } -fn extract_stdio(pipeline: PipelineData) -> (PipelineData, Stdio) { - match pipeline { - PipelineData::ByteStream(stream, metadata) => match stream.into_stdio() { - Ok(pipe) => (PipelineData::Empty, pipe), - Err(stream) => (PipelineData::ByteStream(stream, metadata), Stdio::piped()), - }, - PipelineData::Empty => (PipelineData::Empty, Stdio::inherit()), - data => (data, Stdio::piped()), +/// Expand ndots, but only if it looks like it probably contains them, because there is some lossy +/// path normalization that happens. +fn expand_ndots_safe(path: impl AsRef) -> PathBuf { + let string = path.as_ref().to_string_lossy(); + + // Use ndots if it contains at least `...`, since that's the minimum trigger point, and don't + // use it if it contains ://, because that looks like a URL scheme and the path normalization + // will mess with that. + if string.contains("...") && !string.contains("://") { + expand_ndots(path) + } else { + path.as_ref().to_owned() } } #[cfg(test)] mod test { use super::*; + use nu_test_support::{fs::Stub, playground::Playground}; #[test] - fn remove_quotes_argument_with_equal_test() { - let input = r#"--file="my_file.txt""#.into(); - let res = remove_quotes(input); + fn test_expand_glob() { + Playground::setup("test_expand_glob", |dirs, play| { + play.with_files(&[Stub::EmptyFile("a.txt"), Stub::EmptyFile("b.txt")]); - assert_eq!("--file=my_file.txt", res) + let cwd = dirs.test(); + + let actual = expand_glob("*.txt", cwd, Span::unknown(), &Signals::empty()).unwrap(); + let expected = &["a.txt", "b.txt"]; + assert_eq!(actual, expected); + + let actual = expand_glob("./*.txt", cwd, Span::unknown(), &Signals::empty()).unwrap(); + assert_eq!(actual, expected); + + let actual = expand_glob("'*.txt'", cwd, Span::unknown(), &Signals::empty()).unwrap(); + let expected = &["'*.txt'"]; + assert_eq!(actual, expected); + + let actual = expand_glob(".", cwd, Span::unknown(), &Signals::empty()).unwrap(); + let expected = &["."]; + assert_eq!(actual, expected); + + let actual = expand_glob("./a.txt", cwd, Span::unknown(), &Signals::empty()).unwrap(); + let expected = &["./a.txt"]; + assert_eq!(actual, expected); + + let actual = expand_glob("[*.txt", cwd, Span::unknown(), &Signals::empty()).unwrap(); + let expected = &["[*.txt"]; + assert_eq!(actual, expected); + + let actual = expand_glob("~/foo.txt", cwd, Span::unknown(), &Signals::empty()).unwrap(); + let home = dirs_next::home_dir().expect("failed to get home dir"); + let expected: Vec = vec![home.join("foo.txt").into()]; + assert_eq!(actual, expected); + }) } #[test] - fn argument_without_equal_test() { - let input = r#"--file "my_file.txt""#.into(); - let res = remove_quotes(input); + fn test_write_pipeline_data() { + let engine_state = EngineState::new(); + let stack = Stack::new(); - assert_eq!(r#"--file "my_file.txt""#, res) - } + let mut buf = vec![]; + let input = PipelineData::Empty; + write_pipeline_data(engine_state.clone(), stack.clone(), input, &mut buf).unwrap(); + assert_eq!(buf, b""); - #[test] - fn remove_quotes_argument_with_single_quotes_test() { - let input = r#"--file='my_file.txt'"#.into(); - let res = remove_quotes(input); + let mut buf = vec![]; + let input = PipelineData::Value(Value::string("foo", Span::unknown()), None); + write_pipeline_data(engine_state.clone(), stack.clone(), input, &mut buf).unwrap(); + assert_eq!(buf, b"foo"); - assert_eq!("--file=my_file.txt", res) - } + let mut buf = vec![]; + let input = PipelineData::Value(Value::binary(b"foo", Span::unknown()), None); + write_pipeline_data(engine_state.clone(), stack.clone(), input, &mut buf).unwrap(); + assert_eq!(buf, b"foo"); - #[test] - fn argument_with_inner_quotes_test() { - let input = r#"sh -c 'echo a'"#.into(); - let res = remove_quotes(input); - - assert_eq!("sh -c 'echo a'", res) + let mut buf = vec![]; + let input = PipelineData::ByteStream( + ByteStream::read( + b"foo".as_slice(), + Span::unknown(), + Signals::empty(), + ByteStreamType::Unknown, + ), + None, + ); + write_pipeline_data(engine_state.clone(), stack.clone(), input, &mut buf).unwrap(); + assert_eq!(buf, b"foo"); } } diff --git a/crates/nu-command/src/system/sys/cpu.rs b/crates/nu-command/src/system/sys/cpu.rs index b2f6d6afa6..d24197bf70 100644 --- a/crates/nu-command/src/system/sys/cpu.rs +++ b/crates/nu-command/src/system/sys/cpu.rs @@ -1,4 +1,6 @@ +use super::trim_cstyle_null; use nu_engine::command_prelude::*; +use sysinfo::{CpuRefreshKind, System, MINIMUM_CPU_UPDATE_INTERVAL}; #[derive(Clone)] pub struct SysCpu; @@ -26,7 +28,7 @@ impl Command for SysCpu { call: &Call, _input: PipelineData, ) -> Result { - Ok(super::cpu(call.head).into_pipeline_data()) + Ok(cpu(call.head).into_pipeline_data()) } fn examples(&self) -> Vec { @@ -37,3 +39,42 @@ impl Command for SysCpu { }] } } + +fn cpu(span: Span) -> Value { + let mut sys = System::new(); + sys.refresh_cpu_specifics(CpuRefreshKind::everything()); + // We must refresh the CPU twice a while apart to get valid usage data. + // In theory we could just sleep MINIMUM_CPU_UPDATE_INTERVAL, but I've noticed that + // that gives poor results (error of ~5%). Decided to wait 2x that long, somewhat arbitrarily + std::thread::sleep(MINIMUM_CPU_UPDATE_INTERVAL * 2); + sys.refresh_cpu_specifics(CpuRefreshKind::new().with_cpu_usage()); + + let cpus = sys + .cpus() + .iter() + .map(|cpu| { + // sysinfo CPU usage numbers are not very precise unless you wait a long time between refreshes. + // Round to 1DP (chosen somewhat arbitrarily) so people aren't misled by high-precision floats. + let rounded_usage = (cpu.cpu_usage() * 10.0).round() / 10.0; + + let load_avg = System::load_average(); + let load_avg = format!( + "{:.2}, {:.2}, {:.2}", + load_avg.one, load_avg.five, load_avg.fifteen + ); + + let record = record! { + "name" => Value::string(trim_cstyle_null(cpu.name()), span), + "brand" => Value::string(trim_cstyle_null(cpu.brand()), span), + "freq" => Value::int(cpu.frequency() as i64, span), + "cpu_usage" => Value::float(rounded_usage.into(), span), + "load_average" => Value::string(load_avg, span), + "vendor_id" => Value::string(trim_cstyle_null(cpu.vendor_id()), span), + }; + + Value::record(record, span) + }) + .collect(); + + Value::list(cpus, span) +} diff --git a/crates/nu-command/src/system/sys/disks.rs b/crates/nu-command/src/system/sys/disks.rs index 66991c9087..0d9ce09db3 100644 --- a/crates/nu-command/src/system/sys/disks.rs +++ b/crates/nu-command/src/system/sys/disks.rs @@ -1,4 +1,6 @@ +use super::trim_cstyle_null; use nu_engine::command_prelude::*; +use sysinfo::Disks; #[derive(Clone)] pub struct SysDisks; @@ -26,7 +28,7 @@ impl Command for SysDisks { call: &Call, _input: PipelineData, ) -> Result { - Ok(super::disks(call.head).into_pipeline_data()) + Ok(disks(call.head).into_pipeline_data()) } fn examples(&self) -> Vec { @@ -37,3 +39,27 @@ impl Command for SysDisks { }] } } + +fn disks(span: Span) -> Value { + let disks = Disks::new_with_refreshed_list() + .iter() + .map(|disk| { + let device = trim_cstyle_null(disk.name().to_string_lossy()); + let typ = trim_cstyle_null(disk.file_system().to_string_lossy()); + + let record = record! { + "device" => Value::string(device, span), + "type" => Value::string(typ, span), + "mount" => Value::string(disk.mount_point().to_string_lossy(), span), + "total" => Value::filesize(disk.total_space() as i64, span), + "free" => Value::filesize(disk.available_space() as i64, span), + "removable" => Value::bool(disk.is_removable(), span), + "kind" => Value::string(disk.kind().to_string(), span), + }; + + Value::record(record, span) + }) + .collect(); + + Value::list(disks, span) +} diff --git a/crates/nu-command/src/system/sys/host.rs b/crates/nu-command/src/system/sys/host.rs index 969f59ef99..64dd43424d 100644 --- a/crates/nu-command/src/system/sys/host.rs +++ b/crates/nu-command/src/system/sys/host.rs @@ -1,4 +1,7 @@ +use super::trim_cstyle_null; +use chrono::{DateTime, FixedOffset, Local}; use nu_engine::command_prelude::*; +use sysinfo::System; #[derive(Clone)] pub struct SysHost; @@ -26,8 +29,7 @@ impl Command for SysHost { call: &Call, _input: PipelineData, ) -> Result { - let host = super::host(call.head); - Ok(Value::record(host, call.head).into_pipeline_data()) + Ok(host(call.head).into_pipeline_data()) } fn examples(&self) -> Vec { @@ -38,3 +40,53 @@ impl Command for SysHost { }] } } + +fn host(span: Span) -> Value { + let mut record = Record::new(); + + if let Some(name) = System::name() { + record.push("name", Value::string(trim_cstyle_null(name), span)); + } + if let Some(version) = System::os_version() { + record.push("os_version", Value::string(trim_cstyle_null(version), span)); + } + if let Some(long_version) = System::long_os_version() { + record.push( + "long_os_version", + Value::string(trim_cstyle_null(long_version), span), + ); + } + if let Some(version) = System::kernel_version() { + record.push( + "kernel_version", + Value::string(trim_cstyle_null(version), span), + ); + } + if let Some(hostname) = System::host_name() { + record.push("hostname", Value::string(trim_cstyle_null(hostname), span)); + } + + let uptime = System::uptime() + .saturating_mul(1_000_000_000) + .try_into() + .unwrap_or(i64::MAX); + + record.push("uptime", Value::duration(uptime, span)); + + let boot_time = boot_time() + .map(|time| Value::date(time, span)) + .unwrap_or(Value::nothing(span)); + + record.push("boot_time", boot_time); + + Value::record(record, span) +} + +fn boot_time() -> Option> { + // Broken systems can apparently return really high values. + // See: https://github.com/nushell/nushell/issues/10155 + // First, try to convert u64 to i64, and then try to create a `DateTime`. + let secs = System::boot_time().try_into().ok()?; + let time = DateTime::from_timestamp(secs, 0)?; + Some(time.with_timezone(&Local).fixed_offset()) +} diff --git a/crates/nu-command/src/system/sys/mem.rs b/crates/nu-command/src/system/sys/mem.rs index e5dc36e1b7..89527807d7 100644 --- a/crates/nu-command/src/system/sys/mem.rs +++ b/crates/nu-command/src/system/sys/mem.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use sysinfo::System; #[derive(Clone)] pub struct SysMem; @@ -26,7 +27,7 @@ impl Command for SysMem { call: &Call, _input: PipelineData, ) -> Result { - Ok(super::mem(call.head).into_pipeline_data()) + Ok(mem(call.head).into_pipeline_data()) } fn examples(&self) -> Vec { @@ -37,3 +38,20 @@ impl Command for SysMem { }] } } + +fn mem(span: Span) -> Value { + let mut sys = System::new(); + sys.refresh_memory(); + + let record = record! { + "total" => Value::filesize(sys.total_memory() as i64, span), + "free" => Value::filesize(sys.free_memory() as i64, span), + "used" => Value::filesize(sys.used_memory() as i64, span), + "available" => Value::filesize(sys.available_memory() as i64, span), + "swap total" => Value::filesize(sys.total_swap() as i64, span), + "swap free" => Value::filesize(sys.free_swap() as i64, span), + "swap used" => Value::filesize(sys.used_swap() as i64, span), + }; + + Value::record(record, span) +} diff --git a/crates/nu-command/src/system/sys/mod.rs b/crates/nu-command/src/system/sys/mod.rs index fcb40fd37d..02d271537e 100644 --- a/crates/nu-command/src/system/sys/mod.rs +++ b/crates/nu-command/src/system/sys/mod.rs @@ -16,202 +16,6 @@ pub use sys_::Sys; pub use temp::SysTemp; pub use users::SysUsers; -use chrono::{DateTime, FixedOffset, Local}; -use nu_protocol::{record, Record, Span, Value}; -use sysinfo::{ - Components, CpuRefreshKind, Disks, Networks, System, Users, MINIMUM_CPU_UPDATE_INTERVAL, -}; - -pub fn trim_cstyle_null(s: impl AsRef) -> String { +fn trim_cstyle_null(s: impl AsRef) -> String { s.as_ref().trim_matches('\0').into() } - -pub fn disks(span: Span) -> Value { - let disks = Disks::new_with_refreshed_list() - .iter() - .map(|disk| { - let device = trim_cstyle_null(disk.name().to_string_lossy()); - let typ = trim_cstyle_null(disk.file_system().to_string_lossy()); - - let record = record! { - "device" => Value::string(device, span), - "type" => Value::string(typ, span), - "mount" => Value::string(disk.mount_point().to_string_lossy(), span), - "total" => Value::filesize(disk.total_space() as i64, span), - "free" => Value::filesize(disk.available_space() as i64, span), - "removable" => Value::bool(disk.is_removable(), span), - "kind" => Value::string(disk.kind().to_string(), span), - }; - - Value::record(record, span) - }) - .collect(); - - Value::list(disks, span) -} - -pub fn net(span: Span) -> Value { - let networks = Networks::new_with_refreshed_list() - .iter() - .map(|(iface, data)| { - let record = record! { - "name" => Value::string(trim_cstyle_null(iface), span), - "sent" => Value::filesize(data.total_transmitted() as i64, span), - "recv" => Value::filesize(data.total_received() as i64, span), - }; - - Value::record(record, span) - }) - .collect(); - - Value::list(networks, span) -} - -pub fn cpu(span: Span) -> Value { - let mut sys = System::new(); - sys.refresh_cpu_specifics(CpuRefreshKind::everything()); - // We must refresh the CPU twice a while apart to get valid usage data. - // In theory we could just sleep MINIMUM_CPU_UPDATE_INTERVAL, but I've noticed that - // that gives poor results (error of ~5%). Decided to wait 2x that long, somewhat arbitrarily - std::thread::sleep(MINIMUM_CPU_UPDATE_INTERVAL * 2); - sys.refresh_cpu_specifics(CpuRefreshKind::new().with_cpu_usage()); - - let cpus = sys - .cpus() - .iter() - .map(|cpu| { - // sysinfo CPU usage numbers are not very precise unless you wait a long time between refreshes. - // Round to 1DP (chosen somewhat arbitrarily) so people aren't misled by high-precision floats. - let rounded_usage = (cpu.cpu_usage() * 10.0).round() / 10.0; - - let load_avg = System::load_average(); - let load_avg = format!( - "{:.2}, {:.2}, {:.2}", - load_avg.one, load_avg.five, load_avg.fifteen - ); - - let record = record! { - "name" => Value::string(trim_cstyle_null(cpu.name()), span), - "brand" => Value::string(trim_cstyle_null(cpu.brand()), span), - "freq" => Value::int(cpu.frequency() as i64, span), - "cpu_usage" => Value::float(rounded_usage.into(), span), - "load_average" => Value::string(load_avg, span), - "vendor_id" => Value::string(trim_cstyle_null(cpu.vendor_id()), span), - }; - - Value::record(record, span) - }) - .collect(); - - Value::list(cpus, span) -} - -pub fn mem(span: Span) -> Value { - let mut sys = System::new(); - sys.refresh_memory(); - - let record = record! { - "total" => Value::filesize(sys.total_memory() as i64, span), - "free" => Value::filesize(sys.free_memory() as i64, span), - "used" => Value::filesize(sys.used_memory() as i64, span), - "available" => Value::filesize(sys.available_memory() as i64, span), - "swap total" => Value::filesize(sys.total_swap() as i64, span), - "swap free" => Value::filesize(sys.free_swap() as i64, span), - "swap used" => Value::filesize(sys.used_swap() as i64, span), - }; - - Value::record(record, span) -} - -pub fn users(span: Span) -> Value { - let users = Users::new_with_refreshed_list() - .iter() - .map(|user| { - let groups = user - .groups() - .iter() - .map(|group| Value::string(trim_cstyle_null(group.name()), span)) - .collect(); - - let record = record! { - "name" => Value::string(trim_cstyle_null(user.name()), span), - "groups" => Value::list(groups, span), - }; - - Value::record(record, span) - }) - .collect(); - - Value::list(users, span) -} - -pub fn host(span: Span) -> Record { - let mut record = Record::new(); - - if let Some(name) = System::name() { - record.push("name", Value::string(trim_cstyle_null(name), span)); - } - if let Some(version) = System::os_version() { - record.push("os_version", Value::string(trim_cstyle_null(version), span)); - } - if let Some(long_version) = System::long_os_version() { - record.push( - "long_os_version", - Value::string(trim_cstyle_null(long_version), span), - ); - } - if let Some(version) = System::kernel_version() { - record.push( - "kernel_version", - Value::string(trim_cstyle_null(version), span), - ); - } - if let Some(hostname) = System::host_name() { - record.push("hostname", Value::string(trim_cstyle_null(hostname), span)); - } - - let uptime = System::uptime() - .saturating_mul(1_000_000_000) - .try_into() - .unwrap_or(i64::MAX); - - record.push("uptime", Value::duration(uptime, span)); - - let boot_time = boot_time() - .map(|time| Value::date(time, span)) - .unwrap_or(Value::nothing(span)); - - record.push("boot_time", boot_time); - - record -} - -fn boot_time() -> Option> { - // Broken systems can apparently return really high values. - // See: https://github.com/nushell/nushell/issues/10155 - // First, try to convert u64 to i64, and then try to create a `DateTime`. - let secs = System::boot_time().try_into().ok()?; - let time = DateTime::from_timestamp(secs, 0)?; - Some(time.with_timezone(&Local).fixed_offset()) -} - -pub fn temp(span: Span) -> Value { - let components = Components::new_with_refreshed_list() - .iter() - .map(|component| { - let mut record = record! { - "unit" => Value::string(component.label(), span), - "temp" => Value::float(component.temperature().into(), span), - "high" => Value::float(component.max().into(), span), - }; - - if let Some(critical) = component.critical() { - record.push("critical", Value::float(critical.into(), span)); - } - - Value::record(record, span) - }) - .collect(); - - Value::list(components, span) -} diff --git a/crates/nu-command/src/system/sys/net.rs b/crates/nu-command/src/system/sys/net.rs index 98dca1bc2f..ef1c595800 100644 --- a/crates/nu-command/src/system/sys/net.rs +++ b/crates/nu-command/src/system/sys/net.rs @@ -1,4 +1,6 @@ +use super::trim_cstyle_null; use nu_engine::command_prelude::*; +use sysinfo::Networks; #[derive(Clone)] pub struct SysNet; @@ -26,7 +28,7 @@ impl Command for SysNet { call: &Call, _input: PipelineData, ) -> Result { - Ok(super::net(call.head).into_pipeline_data()) + Ok(net(call.head).into_pipeline_data()) } fn examples(&self) -> Vec { @@ -37,3 +39,20 @@ impl Command for SysNet { }] } } + +fn net(span: Span) -> Value { + let networks = Networks::new_with_refreshed_list() + .iter() + .map(|(iface, data)| { + let record = record! { + "name" => Value::string(trim_cstyle_null(iface), span), + "sent" => Value::filesize(data.total_transmitted() as i64, span), + "recv" => Value::filesize(data.total_received() as i64, span), + }; + + Value::record(record, span) + }) + .collect(); + + Value::list(networks, span) +} diff --git a/crates/nu-command/src/system/sys/sys_.rs b/crates/nu-command/src/system/sys/sys_.rs index 2886836be9..5369064f50 100644 --- a/crates/nu-command/src/system/sys/sys_.rs +++ b/crates/nu-command/src/system/sys/sys_.rs @@ -1,4 +1,4 @@ -use nu_engine::command_prelude::*; +use nu_engine::{command_prelude::*, get_full_help}; #[derive(Clone)] pub struct Sys; @@ -20,41 +20,17 @@ impl Command for Sys { } fn extra_usage(&self) -> &str { - "Note that this command may take a noticeable amount of time to run. To reduce the time taken, you can use the various `sys` sub commands to get the subset of information you are interested in." + "You must use one of the following subcommands. Using this command as-is will only produce this help message." } fn run( &self, engine_state: &EngineState, - _stack: &mut Stack, + stack: &mut Stack, call: &Call, _input: PipelineData, ) -> Result { - nu_protocol::report_error_new( - engine_state, - &ShellError::GenericError { - error: "Deprecated command".into(), - msg: "the `sys` command is deprecated, please use the new subcommands (`sys host`, `sys mem`, etc.)." - .into(), - span: Some(call.head), - help: None, - inner: vec![], - }, - ); - - let head = call.head; - - let mut host = super::host(head); - host.push("sessions", super::users(head)); - let record = record! { - "host" => Value::record(host, head), - "cpu" => super::cpu(head), - "disks" => super::disks(head), - "mem" => super::mem(head), - "temp" => super::temp(head), - "net" => super::net(head), - }; - Ok(Value::record(record, head).into_pipeline_data()) + Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data()) } fn examples(&self) -> Vec { diff --git a/crates/nu-command/src/system/sys/temp.rs b/crates/nu-command/src/system/sys/temp.rs index eaf1ed05db..088471f0fd 100644 --- a/crates/nu-command/src/system/sys/temp.rs +++ b/crates/nu-command/src/system/sys/temp.rs @@ -1,4 +1,5 @@ use nu_engine::command_prelude::*; +use sysinfo::Components; #[derive(Clone)] pub struct SysTemp; @@ -30,7 +31,7 @@ impl Command for SysTemp { call: &Call, _input: PipelineData, ) -> Result { - Ok(super::temp(call.head).into_pipeline_data()) + Ok(temp(call.head).into_pipeline_data()) } fn examples(&self) -> Vec { @@ -41,3 +42,24 @@ impl Command for SysTemp { }] } } + +fn temp(span: Span) -> Value { + let components = Components::new_with_refreshed_list() + .iter() + .map(|component| { + let mut record = record! { + "unit" => Value::string(component.label(), span), + "temp" => Value::float(component.temperature().into(), span), + "high" => Value::float(component.max().into(), span), + }; + + if let Some(critical) = component.critical() { + record.push("critical", Value::float(critical.into(), span)); + } + + Value::record(record, span) + }) + .collect(); + + Value::list(components, span) +} diff --git a/crates/nu-command/src/system/sys/users.rs b/crates/nu-command/src/system/sys/users.rs index 9aab2b9b7b..04d9b9db91 100644 --- a/crates/nu-command/src/system/sys/users.rs +++ b/crates/nu-command/src/system/sys/users.rs @@ -1,4 +1,6 @@ +use super::trim_cstyle_null; use nu_engine::command_prelude::*; +use sysinfo::Users; #[derive(Clone)] pub struct SysUsers; @@ -11,7 +13,7 @@ impl Command for SysUsers { fn signature(&self) -> Signature { Signature::build("sys users") .category(Category::System) - .input_output_types(vec![(Type::Nothing, Type::record())]) + .input_output_types(vec![(Type::Nothing, Type::table())]) } fn usage(&self) -> &str { @@ -25,7 +27,7 @@ impl Command for SysUsers { call: &Call, _input: PipelineData, ) -> Result { - Ok(super::users(call.head).into_pipeline_data()) + Ok(users(call.head).into_pipeline_data()) } fn examples(&self) -> Vec { @@ -36,3 +38,25 @@ impl Command for SysUsers { }] } } + +fn users(span: Span) -> Value { + let users = Users::new_with_refreshed_list() + .iter() + .map(|user| { + let groups = user + .groups() + .iter() + .map(|group| Value::string(trim_cstyle_null(group.name()), span)) + .collect(); + + let record = record! { + "name" => Value::string(trim_cstyle_null(user.name()), span), + "groups" => Value::list(groups, span), + }; + + Value::record(record, span) + }) + .collect(); + + Value::list(users, span) +} diff --git a/crates/nu-command/src/system/uname.rs b/crates/nu-command/src/system/uname.rs index e267fcaeb2..0bcb749f02 100644 --- a/crates/nu-command/src/system/uname.rs +++ b/crates/nu-command/src/system/uname.rs @@ -1,10 +1,5 @@ -use nu_protocol::record; -use nu_protocol::Value; -use nu_protocol::{ - ast::Call, - engine::{Command, EngineState, Stack}, - Category, Example, PipelineData, ShellError, Signature, Type, -}; +use nu_engine::command_prelude::*; +use nu_protocol::{record, Value}; #[derive(Clone)] pub struct UName; diff --git a/crates/nu-command/src/system/which_.rs b/crates/nu-command/src/system/which_.rs index 81b6057864..fd5b0beb18 100644 --- a/crates/nu-command/src/system/which_.rs +++ b/crates/nu-command/src/system/which_.rs @@ -92,7 +92,6 @@ fn get_entries_in_nu( all_entries } -#[cfg(feature = "which-support")] fn get_first_entry_in_path( item: &str, span: Span, @@ -104,17 +103,6 @@ fn get_first_entry_in_path( .ok() } -#[cfg(not(feature = "which-support"))] -fn get_first_entry_in_path( - _item: &str, - _span: Span, - _cwd: impl AsRef, - _paths: impl AsRef, -) -> Option { - None -} - -#[cfg(feature = "which-support")] fn get_all_entries_in_path( item: &str, span: Span, @@ -129,16 +117,6 @@ fn get_all_entries_in_path( .unwrap_or_default() } -#[cfg(not(feature = "which-support"))] -fn get_all_entries_in_path( - _item: &str, - _span: Span, - _cwd: impl AsRef, - _paths: impl AsRef, -) -> Vec { - vec![] -} - #[derive(Debug)] struct WhichArgs { applications: Vec>, @@ -210,7 +188,6 @@ fn which( applications: call.rest(engine_state, stack, 0)?, all: call.has_flag(engine_state, stack, "all")?, }; - let ctrlc = engine_state.ctrlc.clone(); if which_args.applications.is_empty() { return Err(ShellError::MissingParameter { @@ -236,7 +213,9 @@ fn which( output.extend(values); } - Ok(output.into_iter().into_pipeline_data(head, ctrlc)) + Ok(output + .into_iter() + .into_pipeline_data(head, engine_state.signals().clone())) } #[cfg(test)] diff --git a/crates/nu-command/src/viewers/griddle.rs b/crates/nu-command/src/viewers/griddle.rs index e0856e0a70..cd73c1ca71 100644 --- a/crates/nu-command/src/viewers/griddle.rs +++ b/crates/nu-command/src/viewers/griddle.rs @@ -5,6 +5,7 @@ use nu_engine::{command_prelude::*, env_to_string}; use nu_protocol::Config; use nu_term_grid::grid::{Alignment, Cell, Direction, Filling, Grid, GridOptions}; use nu_utils::get_ls_colors; +use std::path::Path; use terminal_size::{Height, Width}; #[derive(Clone)] @@ -67,6 +68,7 @@ prints out the list properly."# }; let use_grid_icons = config.use_grid_icons; let use_color: bool = color_param && config.use_ansi_coloring; + let cwd = engine_state.cwd(Some(stack))?; match input { PipelineData::Value(Value::List { vals, .. }, ..) => { @@ -81,6 +83,7 @@ prints out the list properly."# separator_param, env_str, use_grid_icons, + cwd.as_ref(), )?) } else { Ok(PipelineData::empty()) @@ -98,6 +101,7 @@ prints out the list properly."# separator_param, env_str, use_grid_icons, + cwd.as_ref(), )?) } else { // dbg!(data); @@ -120,6 +124,7 @@ prints out the list properly."# separator_param, env_str, use_grid_icons, + cwd.as_ref(), )?) } x => { @@ -161,6 +166,7 @@ prints out the list properly."# } } +#[allow(clippy::too_many_arguments)] fn create_grid_output( items: Vec<(usize, String, String)>, call: &Call, @@ -169,6 +175,7 @@ fn create_grid_output( separator_param: Option, env_str: Option, use_grid_icons: bool, + cwd: &Path, ) -> Result { let ls_colors = get_ls_colors(env_str); @@ -196,8 +203,8 @@ fn create_grid_output( if use_color { if use_grid_icons { let no_ansi = nu_utils::strip_ansi_unlikely(&value); - let path = std::path::Path::new(no_ansi.as_ref()); - let icon = icon_for_file(path, call.head)?; + let path = cwd.join(no_ansi.as_ref()); + let icon = icon_for_file(&path, call.head)?; let ls_colors_style = ls_colors.style_for_path(path); let icon_style = match ls_colors_style { diff --git a/crates/nu-command/src/viewers/table.rs b/crates/nu-command/src/viewers/table.rs index 26b8c921c5..190c3659af 100644 --- a/crates/nu-command/src/viewers/table.rs +++ b/crates/nu-command/src/viewers/table.rs @@ -5,8 +5,9 @@ use lscolors::{LsColors, Style}; use nu_color_config::{color_from_hex, StyleComputer, TextStyle}; use nu_engine::{command_prelude::*, env::get_config, env_to_string}; +use nu_pretty_hex::HexConfig; use nu_protocol::{ - ByteStream, Config, DataSource, ListStream, PipelineMetadata, TableMode, ValueIterator, + ByteStream, Config, DataSource, ListStream, PipelineMetadata, Signals, TableMode, ValueIterator, }; use nu_table::{ common::create_nu_table_config, CollapsedTable, ExpandedTable, JustTable, NuTable, NuTableCell, @@ -15,10 +16,9 @@ use nu_table::{ use nu_utils::get_ls_colors; use std::{ collections::VecDeque, - io::{Cursor, IsTerminal}, + io::{IsTerminal, Read}, path::PathBuf, str::FromStr, - sync::{atomic::AtomicBool, Arc}, time::Instant, }; use terminal_size::{Height, Width}; @@ -169,7 +169,10 @@ impl Command for Table { }), Value::test_record(record! { "a" => Value::test_int(3), - "b" => Value::test_int(4), + "b" => Value::test_list(vec![ + Value::test_int(4), + Value::test_int(4), + ]) }), ])), }, @@ -183,7 +186,10 @@ impl Command for Table { }), Value::test_record(record! { "a" => Value::test_int(3), - "b" => Value::test_int(4), + "b" => Value::test_list(vec![ + Value::test_int(4), + Value::test_int(4), + ]) }), ])), }, @@ -338,7 +344,7 @@ fn get_theme_flag( struct CmdInput<'a> { engine_state: &'a EngineState, stack: &'a mut Stack, - call: &'a Call, + call: &'a Call<'a>, data: PipelineData, } @@ -346,7 +352,7 @@ impl<'a> CmdInput<'a> { fn new( engine_state: &'a EngineState, stack: &'a mut Stack, - call: &'a Call, + call: &'a Call<'a>, data: PipelineData, ) -> Self { Self { @@ -364,21 +370,23 @@ fn handle_table_command( ) -> Result { let span = input.data.span().unwrap_or(input.call.head); match input.data { + // Binary streams should behave as if they really are `binary` data, and printed as hex + PipelineData::ByteStream(stream, _) if stream.type_() == ByteStreamType::Binary => Ok( + PipelineData::ByteStream(pretty_hex_stream(stream, input.call.head), None), + ), PipelineData::ByteStream(..) => Ok(input.data), PipelineData::Value(Value::Binary { val, .. }, ..) => { - let bytes = { - let mut str = nu_pretty_hex::pretty_hex(&val); - str.push('\n'); - str.into_bytes() - }; - let ctrlc = input.engine_state.ctrlc.clone(); - let stream = ByteStream::read(Cursor::new(bytes), input.call.head, ctrlc); - Ok(PipelineData::ByteStream(stream, None)) + let signals = input.engine_state.signals().clone(); + let stream = ByteStream::read_binary(val, input.call.head, signals); + Ok(PipelineData::ByteStream( + pretty_hex_stream(stream, input.call.head), + None, + )) } // None of these two receive a StyleComputer because handle_row_stream() can produce it by itself using engine_state and stack. PipelineData::Value(Value::List { vals, .. }, metadata) => { - let ctrlc = input.engine_state.ctrlc.clone(); - let stream = ListStream::new(vals.into_iter(), span, ctrlc); + let signals = input.engine_state.signals().clone(); + let stream = ListStream::new(vals.into_iter(), span, signals); input.data = PipelineData::Empty; handle_row_stream(input, cfg, stream, metadata) @@ -401,8 +409,9 @@ fn handle_table_command( Table.run(input.engine_state, input.stack, input.call, base_pipeline) } PipelineData::Value(Value::Range { val, .. }, metadata) => { - let ctrlc = input.engine_state.ctrlc.clone(); - let stream = ListStream::new(val.into_range_iter(span, ctrlc), span, None); + let signals = input.engine_state.signals().clone(); + let stream = + ListStream::new(val.into_range_iter(span, Signals::empty()), span, signals); input.data = PipelineData::Empty; handle_row_stream(input, cfg, stream, metadata) } @@ -410,6 +419,75 @@ fn handle_table_command( } } +fn pretty_hex_stream(stream: ByteStream, span: Span) -> ByteStream { + let mut cfg = HexConfig { + // We are going to render the title manually first + title: true, + // If building on 32-bit, the stream size might be bigger than a usize + length: stream.known_size().and_then(|sz| sz.try_into().ok()), + ..HexConfig::default() + }; + + // This won't really work for us + debug_assert!(cfg.width > 0, "the default hex config width was zero"); + + let mut read_buf = Vec::with_capacity(cfg.width); + + let mut reader = if let Some(reader) = stream.reader() { + reader + } else { + // No stream to read from + return ByteStream::read_string("".into(), span, Signals::empty()); + }; + + ByteStream::from_fn( + span, + Signals::empty(), + ByteStreamType::String, + move |buffer| { + // Turn the buffer into a String we can write to + let mut write_buf = std::mem::take(buffer); + write_buf.clear(); + // SAFETY: we just truncated it empty + let mut write_buf = unsafe { String::from_utf8_unchecked(write_buf) }; + + // Write the title at the beginning + if cfg.title { + nu_pretty_hex::write_title(&mut write_buf, cfg, true).expect("format error"); + cfg.title = false; + + // Put the write_buf back into buffer + *buffer = write_buf.into_bytes(); + + Ok(true) + } else { + // Read up to `cfg.width` bytes + read_buf.clear(); + (&mut reader) + .take(cfg.width as u64) + .read_to_end(&mut read_buf) + .err_span(span)?; + + if !read_buf.is_empty() { + nu_pretty_hex::hex_write(&mut write_buf, &read_buf, cfg, Some(true)) + .expect("format error"); + write_buf.push('\n'); + + // Advance the address offset for next time + cfg.address_offset += read_buf.len(); + + // Put the write_buf back into buffer + *buffer = write_buf.into_bytes(); + + Ok(true) + } else { + Ok(false) + } + } + }, + ) +} + fn handle_record( input: CmdInput, cfg: TableConfig, @@ -418,8 +496,6 @@ fn handle_record( let config = get_config(input.engine_state, input.stack); let span = input.data.span().unwrap_or(input.call.head); let styles = &StyleComputer::from_config(input.engine_state, input.stack); - let ctrlc = input.engine_state.ctrlc.clone(); - let ctrlc1 = ctrlc.clone(); if record.is_empty() { let value = @@ -444,7 +520,7 @@ fn handle_record( let opts = TableOpts::new( &config, styles, - ctrlc, + input.engine_state.signals(), span, cfg.term_width, indent, @@ -456,7 +532,7 @@ fn handle_record( let result = match result { Some(output) => maybe_strip_color(output, &config), - None => report_unsuccessful_output(ctrlc1, cfg.term_width), + None => report_unsuccessful_output(input.engine_state.signals(), cfg.term_width), }; let val = Value::string(result, span); @@ -464,8 +540,8 @@ fn handle_record( Ok(val.into_pipeline_data()) } -fn report_unsuccessful_output(ctrlc1: Option>, term_width: usize) -> String { - if nu_utils::ctrl_c::was_pressed(&ctrlc1) { +fn report_unsuccessful_output(signals: &Signals, term_width: usize) -> String { + if signals.interrupted() { "".into() } else { // assume this failed because the table was too wide @@ -526,12 +602,11 @@ fn handle_row_stream( stream: ListStream, metadata: Option, ) -> Result { - let ctrlc = input.engine_state.ctrlc.clone(); - let stream = match metadata.as_ref() { // First, `ls` sources: Some(PipelineMetadata { data_source: DataSource::Ls, + .. }) => { let config = get_config(input.engine_state, input.stack); let ls_colors_env_str = match input.stack.get_env_var(input.engine_state, "LS_COLORS") { @@ -563,6 +638,7 @@ fn handle_row_stream( // Next, `to html -l` sources: Some(PipelineMetadata { data_source: DataSource::HtmlThemes, + .. }) => { stream.map(|mut value| { if let Value::Record { val: record, .. } = &mut value { @@ -605,10 +681,14 @@ fn handle_row_stream( // for the values it outputs. Because engine_state is passed in, config doesn't need to. input.engine_state.clone(), input.stack.clone(), - ctrlc.clone(), cfg, ); - let stream = ByteStream::from_result_iter(paginator, input.call.head, None); + let stream = ByteStream::from_result_iter( + paginator, + input.call.head, + Signals::empty(), + ByteStreamType::String, + ); Ok(PipelineData::ByteStream(stream, None)) } @@ -641,7 +721,6 @@ struct PagingTableCreator { stream: ValueIterator, engine_state: EngineState, stack: Stack, - ctrlc: Option>, elements_displayed: usize, reached_end: bool, cfg: TableConfig, @@ -654,7 +733,6 @@ impl PagingTableCreator { stream: ListStream, engine_state: EngineState, stack: Stack, - ctrlc: Option>, cfg: TableConfig, ) -> Self { PagingTableCreator { @@ -662,7 +740,6 @@ impl PagingTableCreator { stream: stream.into_inner(), engine_state, stack, - ctrlc, cfg, elements_displayed: 0, reached_end: false, @@ -714,14 +791,14 @@ impl PagingTableCreator { } fn create_table_opts<'a>( - &self, + &'a self, cfg: &'a Config, style_comp: &'a StyleComputer<'a>, ) -> TableOpts<'a> { TableOpts::new( cfg, style_comp, - self.ctrlc.clone(), + self.engine_state.signals(), self.head, self.cfg.term_width, (cfg.table_indent.left, cfg.table_indent.right), @@ -754,12 +831,15 @@ impl Iterator for PagingTableCreator { match self.cfg.abbreviation { Some(abbr) => { (batch, _, end) = - stream_collect_abbriviated(&mut self.stream, abbr, self.ctrlc.clone()); + stream_collect_abbriviated(&mut self.stream, abbr, self.engine_state.signals()); } None => { // Pull from stream until time runs out or we have enough items - (batch, end) = - stream_collect(&mut self.stream, STREAM_PAGE_SIZE, self.ctrlc.clone()); + (batch, end) = stream_collect( + &mut self.stream, + STREAM_PAGE_SIZE, + self.engine_state.signals(), + ); } } @@ -793,14 +873,19 @@ impl Iterator for PagingTableCreator { self.row_offset += batch_size; let config = get_config(&self.engine_state, &self.stack); - convert_table_to_output(table, &config, &self.ctrlc, self.cfg.term_width) + convert_table_to_output( + table, + &config, + self.engine_state.signals(), + self.cfg.term_width, + ) } } fn stream_collect( stream: impl Iterator, size: usize, - ctrlc: Option>, + signals: &Signals, ) -> (Vec, bool) { let start_time = Instant::now(); let mut end = true; @@ -820,7 +905,7 @@ fn stream_collect( break; } - if nu_utils::ctrl_c::was_pressed(&ctrlc) { + if signals.interrupted() { break; } } @@ -831,7 +916,7 @@ fn stream_collect( fn stream_collect_abbriviated( stream: impl Iterator, size: usize, - ctrlc: Option>, + signals: &Signals, ) -> (Vec, usize, bool) { let mut end = true; let mut read = 0; @@ -854,7 +939,7 @@ fn stream_collect_abbriviated( tail.push_back(item); } - if nu_utils::ctrl_c::was_pressed(&ctrlc) { + if signals.interrupted() { end = false; break; } @@ -986,7 +1071,7 @@ fn create_empty_placeholder( fn convert_table_to_output( table: Result, ShellError>, config: &Config, - ctrlc: &Option>, + signals: &Signals, term_width: usize, ) -> Option, ShellError>> { match table { @@ -999,7 +1084,7 @@ fn convert_table_to_output( Some(Ok(bytes)) } Ok(None) => { - let msg = if nu_utils::ctrl_c::was_pressed(ctrlc) { + let msg = if signals.interrupted() { String::from("") } else { // assume this failed because the table was too wide diff --git a/crates/nu-command/tests/commands/all.rs b/crates/nu-command/tests/commands/all.rs index bcf6f86a45..8ed5a6f34f 100644 --- a/crates/nu-command/tests/commands/all.rs +++ b/crates/nu-command/tests/commands/all.rs @@ -55,7 +55,9 @@ fn checks_if_all_returns_error_with_invalid_command() { "# )); - assert!(actual.err.contains("can't run executable") || actual.err.contains("did you mean")); + assert!( + actual.err.contains("Command `st` not found") && actual.err.contains("Did you mean `ast`?") + ); } #[test] diff --git a/crates/nu-command/tests/commands/any.rs b/crates/nu-command/tests/commands/any.rs index 0e43f26cfd..f8af25eb61 100644 --- a/crates/nu-command/tests/commands/any.rs +++ b/crates/nu-command/tests/commands/any.rs @@ -41,7 +41,9 @@ fn checks_if_any_returns_error_with_invalid_command() { "# )); - assert!(actual.err.contains("can't run executable") || actual.err.contains("did you mean")); + assert!( + actual.err.contains("Command `st` not found") && actual.err.contains("Did you mean `ast`?") + ); } #[test] diff --git a/crates/nu-command/tests/commands/bytes/collect.rs b/crates/nu-command/tests/commands/bytes/collect.rs new file mode 100644 index 0000000000..768ab16df4 --- /dev/null +++ b/crates/nu-command/tests/commands/bytes/collect.rs @@ -0,0 +1,27 @@ +use nu_test_support::{nu, pipeline}; + +#[test] +fn test_stream() { + let actual = nu!(pipeline( + " + [0x[01] 0x[02] 0x[03] 0x[04]] + | filter {true} + | bytes collect 0x[aa aa] + | encode hex + " + )); + assert_eq!(actual.out, "01AAAA02AAAA03AAAA04"); +} + +#[test] +fn test_stream_type() { + let actual = nu!(pipeline( + " + [0x[01] 0x[02] 0x[03] 0x[04]] + | filter {true} + | bytes collect 0x[00] + | describe -n + " + )); + assert_eq!(actual.out, "binary (stream)"); +} diff --git a/crates/nu-command/tests/commands/bytes/mod.rs b/crates/nu-command/tests/commands/bytes/mod.rs new file mode 100644 index 0000000000..10b2a494f8 --- /dev/null +++ b/crates/nu-command/tests/commands/bytes/mod.rs @@ -0,0 +1 @@ +mod collect; diff --git a/crates/nu-command/tests/commands/cal.rs b/crates/nu-command/tests/commands/cal.rs index 651ab8d3cd..65d6306845 100644 --- a/crates/nu-command/tests/commands/cal.rs +++ b/crates/nu-command/tests/commands/cal.rs @@ -1,8 +1,9 @@ use nu_test_support::{nu, pipeline}; +// Tests against table/structured data #[test] fn cal_full_year() { - let actual = nu!("cal -y --full-year 2010 | first | to json -r"); + let actual = nu!("cal -t -y --full-year 2010 | first | to json -r"); let first_week_2010_json = r#"{"year":2010,"su":null,"mo":null,"tu":null,"we":null,"th":null,"fr":1,"sa":2}"#; @@ -14,7 +15,7 @@ fn cal_full_year() { fn cal_february_2020_leap_year() { let actual = nu!(pipeline( r#" - cal -ym --full-year 2020 --month-names | where month == "february" | to json -r + cal --as-table -ym --full-year 2020 --month-names | where month == "february" | to json -r "# )); @@ -27,7 +28,7 @@ fn cal_february_2020_leap_year() { fn cal_fr_the_thirteenths_in_2015() { let actual = nu!(pipeline( r#" - cal --full-year 2015 | default 0 fr | where fr == 13 | length + cal --as-table --full-year 2015 | default 0 fr | where fr == 13 | length "# )); @@ -38,7 +39,7 @@ fn cal_fr_the_thirteenths_in_2015() { fn cal_rows_in_2020() { let actual = nu!(pipeline( r#" - cal --full-year 2020 | length + cal --as-table --full-year 2020 | length "# )); @@ -49,7 +50,7 @@ fn cal_rows_in_2020() { fn cal_week_day_start_mo() { let actual = nu!(pipeline( r#" - cal --full-year 2020 -m --month-names --week-start mo | where month == january | to json -r + cal --as-table --full-year 2020 -m --month-names --week-start mo | where month == january | to json -r "# )); @@ -62,9 +63,43 @@ fn cal_week_day_start_mo() { fn cal_sees_pipeline_year() { let actual = nu!(pipeline( r#" - cal --full-year 1020 | get mo | first 4 | to json -r + cal --as-table --full-year 1020 | get mo | first 4 | to json -r "# )); assert_eq!(actual.out, "[null,3,10,17]"); } + +// Tests against default string output +#[test] +fn cal_is_string() { + let actual = nu!(pipeline( + r#" + cal | describe + "# + )); + + assert_eq!(actual.out, "string (stream)"); +} + +#[test] +fn cal_year_num_lines() { + let actual = nu!(pipeline( + r#" + cal --full-year 2024 | lines | length + "# + )); + + assert_eq!(actual.out, "68"); +} + +#[test] +fn cal_week_start_string() { + let actual = nu!(pipeline( + r#" + cal --week-start fr | lines | get 1 | split row '│' | get 2 | ansi strip | str trim + "# + )); + + assert_eq!(actual.out, "sa"); +} diff --git a/crates/nu-command/tests/commands/complete.rs b/crates/nu-command/tests/commands/complete.rs index a5a8b6a256..329e95ad71 100644 --- a/crates/nu-command/tests/commands/complete.rs +++ b/crates/nu-command/tests/commands/complete.rs @@ -24,7 +24,7 @@ fn basic_exit_code() { #[test] fn error() { let actual = nu!("not-found | complete"); - assert!(actual.err.contains("executable was not found")); + assert!(actual.err.contains("Command `not-found` not found")); } #[test] diff --git a/crates/nu-command/tests/commands/conversions/into/binary.rs b/crates/nu-command/tests/commands/conversions/into/binary.rs new file mode 100644 index 0000000000..c8fd24df77 --- /dev/null +++ b/crates/nu-command/tests/commands/conversions/into/binary.rs @@ -0,0 +1,13 @@ +use nu_test_support::nu; + +#[test] +fn sets_stream_from_internal_command_as_binary() { + let result = nu!("seq 1 10 | to text | into binary | describe"); + assert_eq!("binary (stream)", result.out); +} + +#[test] +fn sets_stream_from_external_command_as_binary() { + let result = nu!("^nu --testbin cococo | into binary | describe"); + assert_eq!("binary (stream)", result.out); +} diff --git a/crates/nu-command/tests/commands/conversions/into/mod.rs b/crates/nu-command/tests/commands/conversions/into/mod.rs index a7a829445a..ad10199b5c 100644 --- a/crates/nu-command/tests/commands/conversions/into/mod.rs +++ b/crates/nu-command/tests/commands/conversions/into/mod.rs @@ -1 +1,2 @@ +mod binary; mod int; diff --git a/crates/nu-command/tests/commands/detect_columns.rs b/crates/nu-command/tests/commands/detect_columns.rs index d605eb4aa1..296d039b02 100644 --- a/crates/nu-command/tests/commands/detect_columns.rs +++ b/crates/nu-command/tests/commands/detect_columns.rs @@ -72,12 +72,12 @@ drwxr-xr-x 2 root root 4.0K Mar 20 08:28 =(char nl) drwxr-xr-x 4 root root 4.0K Mar 20 08:18 ~(char nl) -rw-r--r-- 1 root root 3.0K Mar 20 07:23 ~asdf(char nl)\""; let expected = "[ -['column0', 'column1', 'column2', 'column3', 'column4', 'column5', 'column8']; -['drwxr-xr-x', '2', 'root', 'root', '4.0K', 'Mar 20 08:28', '='], -['drwxr-xr-x', '4', 'root', 'root', '4.0K', 'Mar 20 08:18', '~'], -['-rw-r--r--', '1', 'root', 'root', '3.0K', 'Mar 20 07:23', '~asdf'] +['column0', 'column1', 'column2', 'column3', 'column4', 'column5', 'column7', 'column8']; +['drwxr-xr-x', '2', 'root', 'root', '4.0K', 'Mar 20', '08:28', '='], +['drwxr-xr-x', '4', 'root', 'root', '4.0K', 'Mar 20', '08:18', '~'], +['-rw-r--r--', '1', 'root', 'root', '3.0K', 'Mar 20', '07:23', '~asdf'] ]"; - let range = "5..7"; + let range = "5..6"; let cmd = format!( "({} | detect columns -c {} -s 1 --no-headers) == {}", pipeline(body), diff --git a/crates/nu-command/tests/commands/error_make.rs b/crates/nu-command/tests/commands/error_make.rs index 0c6908d2aa..f5714c88ac 100644 --- a/crates/nu-command/tests/commands/error_make.rs +++ b/crates/nu-command/tests/commands/error_make.rs @@ -4,8 +4,9 @@ use nu_test_support::nu; fn error_label_works() { let actual = nu!("error make {msg:foo label:{text:unseen}}"); - assert!(actual.err.contains("unseen")); - assert!(actual.err.contains("╰──")); + assert!(actual + .err + .contains("label at line 1, columns 1 to 10: unseen")); } #[test] diff --git a/crates/nu-command/tests/commands/find.rs b/crates/nu-command/tests/commands/find.rs index ed811f2a57..89bf2d9f39 100644 --- a/crates/nu-command/tests/commands/find.rs +++ b/crates/nu-command/tests/commands/find.rs @@ -17,6 +17,16 @@ fn find_with_list_search_with_char() { assert_eq!(actual.out, "[\"\\u001b[37m\\u001b[0m\\u001b[41;37ml\\u001b[0m\\u001b[37marry\\u001b[0m\",\"\\u001b[37mcur\\u001b[0m\\u001b[41;37ml\\u001b[0m\\u001b[37my\\u001b[0m\"]"); } +#[test] +fn find_with_bytestream_search_with_char() { + let actual = + nu!("\"ABC\" | save foo.txt; let res = open foo.txt | find abc; rm foo.txt; $res | get 0"); + assert_eq!( + actual.out, + "\u{1b}[37m\u{1b}[0m\u{1b}[41;37mABC\u{1b}[0m\u{1b}[37m\u{1b}[0m" + ) +} + #[test] fn find_with_list_search_with_number() { let actual = nu!("[1 2 3 4 5] | find 3 | get 0"); diff --git a/crates/nu-command/tests/commands/first.rs b/crates/nu-command/tests/commands/first.rs index e01478f820..23ccda6669 100644 --- a/crates/nu-command/tests/commands/first.rs +++ b/crates/nu-command/tests/commands/first.rs @@ -68,6 +68,20 @@ fn gets_first_byte() { assert_eq!(actual.out, "170"); } +#[test] +fn gets_first_bytes_from_stream() { + let actual = nu!("(1.. | each { 0x[aa bb cc] } | bytes collect | first 2) == 0x[aa bb]"); + + assert_eq!(actual.out, "true"); +} + +#[test] +fn gets_first_byte_from_stream() { + let actual = nu!("1.. | each { 0x[aa bb cc] } | bytes collect | first"); + + assert_eq!(actual.out, "170"); +} + #[test] // covers a situation where `first` used to behave strangely on list input fn works_with_binary_list() { diff --git a/crates/nu-command/tests/commands/format.rs b/crates/nu-command/tests/commands/format.rs index 6b1649c8b5..6084faf1e4 100644 --- a/crates/nu-command/tests/commands/format.rs +++ b/crates/nu-command/tests/commands/format.rs @@ -37,7 +37,7 @@ fn given_fields_can_be_column_paths() { } #[test] -fn can_use_variables() { +fn cant_use_variables() { let actual = nu!( cwd: "tests/fixtures/formats", pipeline( r#" @@ -46,7 +46,8 @@ fn can_use_variables() { "# )); - assert_eq!(actual.out, "nu is a new type of shell"); + // TODO SPAN: This has been removed during SpanId refactor + assert!(actual.err.contains("Removed functionality")); } #[test] @@ -55,7 +56,7 @@ fn error_unmatched_brace() { cwd: "tests/fixtures/formats", pipeline( r#" open cargo_sample.toml - | format pattern "{$it.package.name" + | format pattern "{package.name" "# )); diff --git a/crates/nu-command/tests/commands/into_filesize.rs b/crates/nu-command/tests/commands/into_filesize.rs index 2917e141db..6a72c76b5d 100644 --- a/crates/nu-command/tests/commands/into_filesize.rs +++ b/crates/nu-command/tests/commands/into_filesize.rs @@ -63,8 +63,71 @@ fn into_filesize_negative_filesize() { } #[test] -fn into_negative_filesize() { +fn into_filesize_negative_str_filesize() { + let actual = nu!("'-3kib' | into filesize"); + + assert!(actual.out.contains("-3.0 KiB")); +} + +#[test] +fn into_filesize_wrong_negative_str_filesize() { + let actual = nu!("'--3kib' | into filesize"); + + assert!(actual.err.contains("can't convert string to filesize")); +} + +#[test] +fn into_filesize_large_negative_str_filesize() { + let actual = nu!("'-10000PiB' | into filesize"); + + assert!(actual.err.contains("can't convert string to filesize")); +} + +#[test] +fn into_filesize_negative_str() { let actual = nu!("'-1' | into filesize"); assert!(actual.out.contains("-1 B")); } + +#[test] +fn into_filesize_wrong_negative_str() { + let actual = nu!("'--1' | into filesize"); + + assert!(actual.err.contains("can't convert string to filesize")); +} + +#[test] +fn into_filesize_positive_str_filesize() { + let actual = nu!("'+1Kib' | into filesize"); + + assert!(actual.out.contains("1.0 KiB")); +} + +#[test] +fn into_filesize_wrong_positive_str_filesize() { + let actual = nu!("'++1Kib' | into filesize"); + + assert!(actual.err.contains("can't convert string to filesize")); +} + +#[test] +fn into_filesize_large_positive_str_filesize() { + let actual = nu!("'+10000PiB' | into filesize"); + + assert!(actual.err.contains("can't convert string to filesize")); +} + +#[test] +fn into_filesize_positive_str() { + let actual = nu!("'+1' | into filesize"); + + assert!(actual.out.contains("1 B")); +} + +#[test] +fn into_filesize_wrong_positive_str() { + let actual = nu!("'++1' | into filesize"); + + assert!(actual.err.contains("can't convert string to filesize")); +} diff --git a/crates/nu-command/tests/commands/last.rs b/crates/nu-command/tests/commands/last.rs index b0c67e49be..986b433ea7 100644 --- a/crates/nu-command/tests/commands/last.rs +++ b/crates/nu-command/tests/commands/last.rs @@ -68,6 +68,20 @@ fn gets_last_byte() { assert_eq!(actual.out, "204"); } +#[test] +fn gets_last_bytes_from_stream() { + let actual = nu!("(1..10 | each { 0x[aa bb cc] } | bytes collect | last 2) == 0x[bb cc]"); + + assert_eq!(actual.out, "true"); +} + +#[test] +fn gets_last_byte_from_stream() { + let actual = nu!("1..10 | each { 0x[aa bb cc] } | bytes collect | last"); + + assert_eq!(actual.out, "204"); +} + #[test] fn last_errors_on_negative_index() { let actual = nu!("[1, 2, 3] | last -2"); diff --git a/crates/nu-command/tests/commands/length.rs b/crates/nu-command/tests/commands/length.rs index 63318e65db..b67ac452e8 100644 --- a/crates/nu-command/tests/commands/length.rs +++ b/crates/nu-command/tests/commands/length.rs @@ -2,7 +2,7 @@ use nu_test_support::nu; #[test] fn length_columns_in_cal_table() { - let actual = nu!("cal | columns | length"); + let actual = nu!("cal --as-table | columns | length"); assert_eq!(actual.out, "7"); } diff --git a/crates/nu-command/tests/commands/mod.rs b/crates/nu-command/tests/commands/mod.rs index d7215e002b..a1cb82a4b4 100644 --- a/crates/nu-command/tests/commands/mod.rs +++ b/crates/nu-command/tests/commands/mod.rs @@ -4,6 +4,7 @@ mod any; mod append; mod assignment; mod break_; +mod bytes; mod cal; mod cd; mod compact; @@ -124,7 +125,6 @@ mod upsert; mod url; mod use_; mod where_; -#[cfg(feature = "which-support")] mod which; mod while_; mod with_env; diff --git a/crates/nu-command/tests/commands/network/http/delete.rs b/crates/nu-command/tests/commands/network/http/delete.rs index bc71ed3945..c284c2ccfd 100644 --- a/crates/nu-command/tests/commands/network/http/delete.rs +++ b/crates/nu-command/tests/commands/network/http/delete.rs @@ -20,6 +20,25 @@ fn http_delete_is_success() { assert!(actual.out.is_empty()) } +#[test] +fn http_delete_is_success_pipeline() { + let mut server = Server::new(); + + let _mock = server.mock("DELETE", "/").create(); + + let actual = nu!(pipeline( + format!( + r#" + "foo" | http delete {url} + "#, + url = server.url() + ) + .as_str() + )); + + assert!(actual.out.is_empty()) +} + #[test] fn http_delete_failed_due_to_server_error() { let mut server = Server::new(); diff --git a/crates/nu-command/tests/commands/network/http/patch.rs b/crates/nu-command/tests/commands/network/http/patch.rs index 4196b304c6..dc3a755baa 100644 --- a/crates/nu-command/tests/commands/network/http/patch.rs +++ b/crates/nu-command/tests/commands/network/http/patch.rs @@ -20,6 +20,25 @@ fn http_patch_is_success() { assert!(actual.out.is_empty()) } +#[test] +fn http_patch_is_success_pipeline() { + let mut server = Server::new(); + + let _mock = server.mock("PATCH", "/").match_body("foo").create(); + + let actual = nu!(pipeline( + format!( + r#" + "foo" | http patch {url} + "#, + url = server.url() + ) + .as_str() + )); + + assert!(actual.out.is_empty()) +} + #[test] fn http_patch_failed_due_to_server_error() { let mut server = Server::new(); @@ -55,7 +74,9 @@ fn http_patch_failed_due_to_missing_body() { .as_str() )); - assert!(actual.err.contains("Usage: http patch")) + assert!(actual + .err + .contains("Data must be provided either through pipeline or positional argument")) } #[test] diff --git a/crates/nu-command/tests/commands/network/http/post.rs b/crates/nu-command/tests/commands/network/http/post.rs index bd13542482..02aa8df23f 100644 --- a/crates/nu-command/tests/commands/network/http/post.rs +++ b/crates/nu-command/tests/commands/network/http/post.rs @@ -19,6 +19,24 @@ fn http_post_is_success() { assert!(actual.out.is_empty()) } +#[test] +fn http_post_is_success_pipeline() { + let mut server = Server::new(); + + let _mock = server.mock("POST", "/").match_body("foo").create(); + + let actual = nu!(pipeline( + format!( + r#" + "foo" | http post {url} + "#, + url = server.url() + ) + .as_str() + )); + + assert!(actual.out.is_empty()) +} #[test] fn http_post_failed_due_to_server_error() { @@ -55,7 +73,9 @@ fn http_post_failed_due_to_missing_body() { .as_str() )); - assert!(actual.err.contains("Usage: http post")) + assert!(actual + .err + .contains("Data must be provided either through pipeline or positional argument")) } #[test] diff --git a/crates/nu-command/tests/commands/network/http/put.rs b/crates/nu-command/tests/commands/network/http/put.rs index 002b1f8632..43251cd21a 100644 --- a/crates/nu-command/tests/commands/network/http/put.rs +++ b/crates/nu-command/tests/commands/network/http/put.rs @@ -20,6 +20,25 @@ fn http_put_is_success() { assert!(actual.out.is_empty()) } +#[test] +fn http_put_is_success_pipeline() { + let mut server = Server::new(); + + let _mock = server.mock("PUT", "/").match_body("foo").create(); + + let actual = nu!(pipeline( + format!( + r#" + "foo" | http put {url} + "#, + url = server.url() + ) + .as_str() + )); + + assert!(actual.out.is_empty()) +} + #[test] fn http_put_failed_due_to_server_error() { let mut server = Server::new(); @@ -55,7 +74,9 @@ fn http_put_failed_due_to_missing_body() { .as_str() )); - assert!(actual.err.contains("Usage: http put")) + assert!(actual + .err + .contains("Data must be provided either through pipeline or positional argument")) } #[test] diff --git a/crates/nu-command/tests/commands/open.rs b/crates/nu-command/tests/commands/open.rs index f857718c72..ac5e2f99e5 100644 --- a/crates/nu-command/tests/commands/open.rs +++ b/crates/nu-command/tests/commands/open.rs @@ -238,22 +238,6 @@ fn parses_xml() { assert_eq!(actual.out, "https://www.jntrnr.com/off-to-new-adventures/") } -#[cfg(feature = "dataframe")] -#[test] -fn parses_arrow_ipc() { - let actual = nu!( - cwd: "tests/fixtures/formats", pipeline( - " - dfr open caco3_plastics.arrow - | dfr into-nu - | first - | get origin - " - )); - - assert_eq!(actual.out, "SPAIN") -} - #[test] fn errors_if_file_not_found() { let actual = nu!( diff --git a/crates/nu-command/tests/commands/path/type_.rs b/crates/nu-command/tests/commands/path/type_.rs index 2317cea060..e67b957e4c 100644 --- a/crates/nu-command/tests/commands/path/type_.rs +++ b/crates/nu-command/tests/commands/path/type_.rs @@ -74,3 +74,14 @@ fn returns_type_of_existing_file_const() { assert_eq!(actual.out, "dir"); }) } + +#[test] +fn respects_cwd() { + Playground::setup("path_type_respects_cwd", |dirs, sandbox| { + sandbox.within("foo").with_files(&[EmptyFile("bar.txt")]); + + let actual = nu!(cwd: dirs.test(), "cd foo; 'bar.txt' | path type"); + + assert_eq!(actual.out, "file"); + }) +} diff --git a/crates/nu-command/tests/commands/run_external.rs b/crates/nu-command/tests/commands/run_external.rs index 14428edcbe..154c31b71c 100644 --- a/crates/nu-command/tests/commands/run_external.rs +++ b/crates/nu-command/tests/commands/run_external.rs @@ -1,4 +1,3 @@ -#[cfg(not(windows))] use nu_test_support::fs::Stub::EmptyFile; use nu_test_support::playground::Playground; use nu_test_support::{nu, pipeline}; @@ -17,7 +16,6 @@ fn better_empty_redirection() { assert!(!actual.out.contains('2')); } -#[cfg(not(windows))] #[test] fn explicit_glob() { Playground::setup("external with explicit glob", |dirs, sandbox| { @@ -30,15 +28,15 @@ fn explicit_glob() { let actual = nu!( cwd: dirs.test(), pipeline( r#" - ^ls | glob '*.txt' | length + ^nu --testbin cococo ('*.txt' | into glob) "# )); - assert_eq!(actual.out, "2"); + assert!(actual.out.contains("D&D_volume_1.txt")); + assert!(actual.out.contains("D&D_volume_2.txt")); }) } -#[cfg(not(windows))] #[test] fn bare_word_expand_path_glob() { Playground::setup("bare word should do the expansion", |dirs, sandbox| { @@ -51,7 +49,7 @@ fn bare_word_expand_path_glob() { let actual = nu!( cwd: dirs.test(), pipeline( " - ^ls *.txt + ^nu --testbin cococo *.txt " )); @@ -60,7 +58,6 @@ fn bare_word_expand_path_glob() { }) } -#[cfg(not(windows))] #[test] fn backtick_expand_path_glob() { Playground::setup("backtick should do the expansion", |dirs, sandbox| { @@ -73,7 +70,7 @@ fn backtick_expand_path_glob() { let actual = nu!( cwd: dirs.test(), pipeline( r#" - ^ls `*.txt` + ^nu --testbin cococo `*.txt` "# )); @@ -82,7 +79,6 @@ fn backtick_expand_path_glob() { }) } -#[cfg(not(windows))] #[test] fn single_quote_does_not_expand_path_glob() { Playground::setup("single quote do not run the expansion", |dirs, sandbox| { @@ -95,15 +91,14 @@ fn single_quote_does_not_expand_path_glob() { let actual = nu!( cwd: dirs.test(), pipeline( r#" - ^ls '*.txt' + ^nu --testbin cococo '*.txt' "# )); - assert!(actual.err.contains("No such file or directory")); + assert_eq!(actual.out, "*.txt"); }) } -#[cfg(not(windows))] #[test] fn double_quote_does_not_expand_path_glob() { Playground::setup("double quote do not run the expansion", |dirs, sandbox| { @@ -116,22 +111,21 @@ fn double_quote_does_not_expand_path_glob() { let actual = nu!( cwd: dirs.test(), pipeline( r#" - ^ls "*.txt" + ^nu --testbin cococo "*.txt" "# )); - assert!(actual.err.contains("No such file or directory")); + assert_eq!(actual.out, "*.txt"); }) } -#[cfg(not(windows))] #[test] fn failed_command_with_semicolon_will_not_execute_following_cmds() { Playground::setup("external failed command with semicolon", |dirs, _| { let actual = nu!( cwd: dirs.test(), pipeline( " - ^ls *.abc; echo done + nu --testbin fail; echo done " )); @@ -155,16 +149,51 @@ fn external_args_with_quoted() { #[cfg(not(windows))] #[test] -fn external_arg_with_long_flag_value_quoted() { - Playground::setup("external failed command with semicolon", |dirs, _| { +fn external_arg_with_option_like_embedded_quotes() { + // TODO: would be nice to make this work with cococo, but arg parsing interferes + Playground::setup( + "external arg with option like embedded quotes", + |dirs, _| { + let actual = nu!( + cwd: dirs.test(), pipeline( + r#" + ^echo --foo='bar' -foo='bar' + "# + )); + + assert_eq!(actual.out, "--foo=bar -foo=bar"); + }, + ) +} + +#[test] +fn external_arg_with_non_option_like_embedded_quotes() { + Playground::setup( + "external arg with non option like embedded quotes", + |dirs, _| { + let actual = nu!( + cwd: dirs.test(), pipeline( + r#" + ^nu --testbin cococo foo='bar' 'foo'=bar + "# + )); + + assert_eq!(actual.out, "foo=bar foo=bar"); + }, + ) +} + +#[test] +fn external_arg_with_string_interpolation() { + Playground::setup("external arg with string interpolation", |dirs, _| { let actual = nu!( cwd: dirs.test(), pipeline( r#" - ^echo --foo='bar' + ^nu --testbin cococo foo=(2 + 2) $"foo=(2 + 2)" foo=$"(2 + 2)" "# )); - assert_eq!(actual.out, "--foo=bar"); + assert_eq!(actual.out, "foo=4 foo=4 foo=4"); }) } @@ -200,6 +229,99 @@ fn external_command_escape_args() { }) } +#[test] +fn external_command_ndots_args() { + let actual = nu!(r#" + nu --testbin cococo foo/. foo/.. foo/... foo/./bar foo/../bar foo/.../bar ./bar ../bar .../bar + "#); + + assert_eq!( + actual.out, + if cfg!(windows) { + // Windows is a bit weird right now, where if ndots has to fix something it's going to + // change everything to backslashes too. Would be good to fix that + r"foo/. foo/.. foo\..\.. foo/./bar foo/../bar foo\..\..\bar ./bar ../bar ..\..\bar" + } else { + r"foo/. foo/.. foo/../.. foo/./bar foo/../bar foo/../../bar ./bar ../bar ../../bar" + } + ); +} + +#[test] +fn external_command_url_args() { + // If ndots is not handled correctly, we can lose the double forward slashes that are needed + // here + let actual = nu!(r#" + nu --testbin cococo http://example.com http://example.com/.../foo //foo + "#); + + assert_eq!( + actual.out, + "http://example.com http://example.com/.../foo //foo" + ); +} + +#[test] +#[cfg_attr( + not(target_os = "linux"), + ignore = "only runs on Linux, where controlling the HOME var is reliable" +)] +fn external_command_expand_tilde() { + Playground::setup("external command expand tilde", |dirs, _| { + // Make a copy of the nu executable that we can use + let mut src = std::fs::File::open(nu_test_support::fs::binaries().join("nu")) + .expect("failed to open nu"); + let mut dst = std::fs::File::create_new(dirs.test().join("test_nu")) + .expect("failed to create test_nu file"); + std::io::copy(&mut src, &mut dst).expect("failed to copy data for nu binary"); + + // Make test_nu have the same permissions so that it's executable + dst.set_permissions( + src.metadata() + .expect("failed to get nu metadata") + .permissions(), + ) + .expect("failed to set permissions on test_nu"); + + // Close the files + drop(dst); + drop(src); + + let actual = nu!( + envs: vec![ + ("HOME".to_string(), dirs.test().to_string_lossy().into_owned()), + ], + r#" + ^~/test_nu --testbin cococo hello + "# + ); + assert_eq!(actual.out, "hello"); + }) +} + +#[test] +fn external_arg_expand_tilde() { + Playground::setup("external arg expand tilde", |dirs, _| { + let actual = nu!( + cwd: dirs.test(), pipeline( + r#" + ^nu --testbin cococo ~/foo ~/(2 + 2) + "# + )); + + let home = dirs_next::home_dir().expect("failed to find home dir"); + + assert_eq!( + actual.out, + format!( + "{} {}", + home.join("foo").display(), + home.join("4").display() + ) + ); + }) +} + #[test] fn external_command_not_expand_tilde_with_quotes() { Playground::setup( @@ -231,21 +353,6 @@ fn external_command_receives_raw_binary_data() { }) } -#[cfg(windows)] -#[test] -fn failed_command_with_semicolon_will_not_execute_following_cmds_windows() { - Playground::setup("external failed command with semicolon", |dirs, _| { - let actual = nu!( - cwd: dirs.test(), pipeline( - " - ^cargo asdf; echo done - " - )); - - assert!(!actual.out.contains("done")); - }) -} - #[cfg(windows)] #[test] fn can_run_batch_files() { diff --git a/crates/nu-command/tests/commands/skip/skip_.rs b/crates/nu-command/tests/commands/skip/skip_.rs index 790c58db4e..c98de496c8 100644 --- a/crates/nu-command/tests/commands/skip/skip_.rs +++ b/crates/nu-command/tests/commands/skip/skip_.rs @@ -1,13 +1,17 @@ use nu_test_support::nu; #[test] -fn binary_skip_will_raise_error() { - let actual = nu!( - cwd: "tests/fixtures/formats", - "open sample_data.ods --raw | skip 2" - ); +fn skips_bytes() { + let actual = nu!("(0x[aa bb cc] | skip 2) == 0x[cc]"); - assert!(actual.err.contains("only_supports_this_input_type")); + assert_eq!(actual.out, "true"); +} + +#[test] +fn skips_bytes_from_stream() { + let actual = nu!("([0 1] | each { 0x[aa bb cc] } | bytes collect | skip 2) == 0x[cc aa bb cc]"); + + assert_eq!(actual.out, "true"); } #[test] diff --git a/crates/nu-command/tests/commands/source_env.rs b/crates/nu-command/tests/commands/source_env.rs index 4e8da55c78..9dd0320a45 100644 --- a/crates/nu-command/tests/commands/source_env.rs +++ b/crates/nu-command/tests/commands/source_env.rs @@ -283,13 +283,17 @@ fn source_env_is_scoped() { let actual = nu!(cwd: dirs.test(), &inp.join("; ")); - assert!(actual.err.contains("executable was not found")); + assert!(actual + .err + .contains("Command `no-name-similar-to-this` not found")); let inp = &[r#"source-env spam.nu"#, r#"nor-similar-to-this"#]; let actual = nu!(cwd: dirs.test(), &inp.join("; ")); - assert!(actual.err.contains("executable was not found")); + assert!(actual + .err + .contains("Command `nor-similar-to-this` not found")); }) } diff --git a/crates/nu-command/tests/commands/str_/collect.rs b/crates/nu-command/tests/commands/str_/join.rs similarity index 65% rename from crates/nu-command/tests/commands/str_/collect.rs rename to crates/nu-command/tests/commands/str_/join.rs index 154ce30537..e04652e810 100644 --- a/crates/nu-command/tests/commands/str_/collect.rs +++ b/crates/nu-command/tests/commands/str_/join.rs @@ -22,6 +22,18 @@ fn test_2() { assert_eq!(actual.out, "abcd"); } +#[test] +fn test_stream() { + let actual = nu!("[a b c d] | filter {true} | str join ."); + assert_eq!(actual.out, "a.b.c.d"); +} + +#[test] +fn test_stream_type() { + let actual = nu!("[a b c d] | filter {true} | str join . | describe -n"); + assert_eq!(actual.out, "string (stream)"); +} + #[test] fn construct_a_path() { let actual = nu!(pipeline( diff --git a/crates/nu-command/tests/commands/str_/mod.rs b/crates/nu-command/tests/commands/str_/mod.rs index 9f1e90e853..2aeabfd04a 100644 --- a/crates/nu-command/tests/commands/str_/mod.rs +++ b/crates/nu-command/tests/commands/str_/mod.rs @@ -1,5 +1,5 @@ -mod collect; mod into_string; +mod join; use nu_test_support::fs::Stub::FileWithContent; use nu_test_support::playground::Playground; @@ -255,7 +255,7 @@ fn substrings_the_input() { } #[test] -fn substring_errors_if_start_index_is_greater_than_end_index() { +fn substring_empty_if_start_index_is_greater_than_end_index() { Playground::setup("str_test_9", |dirs, sandbox| { sandbox.with_files(&[FileWithContent( "sample.toml", @@ -269,13 +269,11 @@ fn substring_errors_if_start_index_is_greater_than_end_index() { cwd: dirs.test(), pipeline( r#" open sample.toml - | str substring 6..5 fortune.teller.phone + | str substring 6..4 fortune.teller.phone + | get fortune.teller.phone "# )); - - assert!(actual - .err - .contains("End must be greater than or equal to Start")) + assert_eq!(actual.out, "") }) } @@ -342,7 +340,7 @@ fn substrings_the_input_and_treats_start_index_as_zero_if_blank_start_index_give cwd: dirs.test(), pipeline( r#" open sample.toml - | str substring ..2 package.name + | str substring ..1 package.name | get package.name "# )); @@ -375,6 +373,21 @@ fn substrings_the_input_and_treats_end_index_as_length_if_blank_end_index_given( }) } +#[test] +fn substring_by_negative_index() { + Playground::setup("str_test_13", |dirs, _| { + let actual = nu!( + cwd: dirs.test(), "'apples' | str substring 0..-1", + ); + assert_eq!(actual.out, "apples"); + + let actual = nu!( + cwd: dirs.test(), "'apples' | str substring 0..<-1", + ); + assert_eq!(actual.out, "apple"); + }) +} + #[test] fn str_reverse() { let actual = nu!(r#" diff --git a/crates/nu-command/tests/commands/table.rs b/crates/nu-command/tests/commands/table.rs index a986c7b15b..16d5eb54ab 100644 --- a/crates/nu-command/tests/commands/table.rs +++ b/crates/nu-command/tests/commands/table.rs @@ -667,106 +667,6 @@ fn test_expand_big_0() { let expected = join_lines([ "╭──────────────────┬───────────────────────────────────────────────────────────╮", - "│ │ ╭───┬─────────┬────────────╮ │", - "│ bench │ │ # │ harness │ name │ │", - "│ │ ├───┼─────────┼────────────┤ │", - "│ │ │ 0 │ false │ benchmarks │ │", - "│ │ ╰───┴─────────┴────────────╯ │", - "│ │ ╭───┬──────┬─────────────╮ │", - "│ bin │ │ # │ name │ path │ │", - "│ │ ├───┼──────┼─────────────┤ │", - "│ │ │ 0 │ nu │ src/main.rs │ │", - "│ │ ╰───┴──────┴─────────────╯ │", - "│ │ ╭───────────────┬───────────────────────────────────────╮ │", - "│ dependencies │ │ │ ╭──────────┬───────────────╮ │ │", - "│ │ │ chrono │ │ │ ╭───┬───────╮ │ │ │", - "│ │ │ │ │ features │ │ 0 │ serde │ │ │ │", - "│ │ │ │ │ │ ╰───┴───────╯ │ │ │", - "│ │ │ │ │ version │ 0.4.23 │ │ │", - "│ │ │ │ ╰──────────┴───────────────╯ │ │", - "│ │ │ crossterm │ 0.24.0 │ │", - "│ │ │ ctrlc │ 3.2.1 │ │", - "│ │ │ is_executable │ 1.0.1 │ │", - "│ │ │ log │ 0.4 │ │", - "│ │ │ │ ╭──────────┬────────────────────────╮ │ │", - "│ │ │ miette │ │ │ ╭───┬────────────────╮ │ │ │", - "│ │ │ │ │ features │ │ 0 │ fancy-no-backt │ │ │ │", - "│ │ │ │ │ │ │ │ race │ │ │ │", - "│ │ │ │ │ │ ╰───┴────────────────╯ │ │ │", - "│ │ │ │ │ version │ 5.5.0 │ │ │", - "│ │ │ │ ╰──────────┴────────────────────────╯ │ │", - "│ │ │ nu-ansi-term │ 0.46.0 │ │", - "│ │ │ │ ╭─────────┬─────────────────╮ │ │", - "│ │ │ nu-cli │ │ path │ ./crates/nu-cli │ │ │", - "│ │ │ │ │ version │ 0.74.1 │ │ │", - "│ │ │ │ ╰─────────┴─────────────────╯ │ │", - "│ │ │ │ ╭────────────┬──────────────────────╮ │ │", - "│ │ │ nu-engine │ │ path │ ./crates/nu-engine │ │ │", - "│ │ │ │ │ version │ 0.74.1 │ │ │", - "│ │ │ │ ╰────────────┴──────────────────────╯ │ │", - "│ │ │ rayon │ 1.6.1 │ │", - "│ │ │ │ ╭─────────────┬─────────────────────╮ │ │", - "│ │ │ reedline │ │ │ ╭───┬──────────╮ │ │ │", - "│ │ │ │ │ features │ │ 0 │ bashisms │ │ │ │", - "│ │ │ │ │ │ │ 1 │ sqlite │ │ │ │", - "│ │ │ │ │ │ ╰───┴──────────╯ │ │ │", - "│ │ │ │ │ version │ 0.14.0 │ │ │", - "│ │ │ │ ╰─────────────┴─────────────────────╯ │ │", - "│ │ │ simplelog │ 0.12.0 │ │", - "│ │ │ time │ 0.3.12 │ │", - "│ │ ╰───────────────┴───────────────────────────────────────╯ │", - "│ │ ╭───────────────────┬───────────────────────────────────╮ │", - "│ dev-dependencies │ │ assert_cmd │ 2.0.2 │ │", - "│ │ │ criterion │ 0.4 │ │", - "│ │ │ hamcrest2 │ 0.3.0 │ │", - "│ │ │ itertools │ 0.10.3 │ │", - "│ │ │ │ ╭─────────┬─────────────────────╮ │ │", - "│ │ │ nu-test-support │ │ path │ ./crates/nu-test-su │ │ │", - "│ │ │ │ │ │ pport │ │ │", - "│ │ │ │ │ version │ 0.74.1 │ │ │", - "│ │ │ │ ╰─────────┴─────────────────────╯ │ │", - "│ │ │ pretty_assertions │ 1.0.0 │ │", - "│ │ │ │ ╭────────────────────┬──────────╮ │ │", - "│ │ │ rstest │ │ default-features │ false │ │ │", - "│ │ │ │ │ version │ 0.15.0 │ │ │", - "│ │ │ │ ╰────────────────────┴──────────╯ │ │", - "│ │ │ serial_test │ 0.10.0 │ │", - "│ │ │ tempfile │ 3.2.0 │ │", - "│ │ ╰───────────────────┴───────────────────────────────────╯ │", - "│ │ ╭─────────────────────┬─────────────────────────────────╮ │", - "│ features │ │ │ ╭───┬───────────────╮ │ │", - "│ │ │ default │ │ 0 │ plugin │ │ │", - "│ │ │ │ │ 1 │ which-support │ │ │", - "│ │ │ │ │ 2 │ trash-support │ │ │", - "│ │ │ │ │ 3 │ sqlite │ │ │", - "│ │ │ │ ╰───┴───────────────╯ │ │", - "│ │ │ │ ╭───┬─────────╮ │ │", - "│ │ │ extra │ │ 0 │ default │ │ │", - "│ │ │ │ ╰───┴─────────╯ │ │", - "│ │ │ │ ╭───┬────────────────────╮ │ │", - "│ │ │ plugin │ │ 0 │ nu-plugin │ │ │", - "│ │ │ │ │ 1 │ nu-cli/plugin │ │ │", - "│ │ │ │ │ 2 │ nu-parser/plugin │ │ │", - "│ │ │ │ │ 3 │ nu-command/plugin │ │ │", - "│ │ │ │ │ 4 │ nu-protocol/plugin │ │ │", - "│ │ │ │ │ 5 │ nu-engine/plugin │ │ │", - "│ │ │ │ ╰───┴────────────────────╯ │ │", - "│ │ │ │ ╭───┬─────────╮ │ │", - "│ │ │ stable │ │ 0 │ default │ │ │", - "│ │ │ │ ╰───┴─────────╯ │ │", - "│ │ │ │ ╭───┬─────────────╮ │ │", - "│ │ │ static-link-openssl │ │ 0 │ dep:openssl │ │ │", - "│ │ │ │ ╰───┴─────────────╯ │ │", - "│ │ │ │ ╭───┬─────────────────────────╮ │ │", - "│ │ │ trash-support │ │ 0 │ nu-command/trash-suppor │ │ │", - "│ │ │ │ │ │ t │ │ │", - "│ │ │ │ ╰───┴─────────────────────────╯ │ │", - "│ │ │ wasi │ [list 0 items] │ │", - "│ │ │ │ ╭───┬─────────────────────────╮ │ │", - "│ │ │ which-support │ │ 0 │ nu-command/which-suppor │ │ │", - "│ │ │ │ │ │ t │ │ │", - "│ │ │ │ ╰───┴─────────────────────────╯ │ │", - "│ │ ╰─────────────────────┴─────────────────────────────────╯ │", "│ │ ╭───────────────┬───────────────────────────────────────╮ │", "│ package │ │ │ ╭───┬───────────────────────────────╮ │ │", "│ │ │ authors │ │ 0 │ The Nushell Project │ │ │", @@ -781,13 +681,13 @@ fn test_expand_big_0() { "│ │ │ │ ╰───┴────────╯ │ │", "│ │ │ homepage │ https://www.nushell.sh │ │", "│ │ │ license │ MIT │ │", + "│ │ │ name │ nu │ │", + "│ │ │ repository │ https://github.com/nushell/nushell │ │", + "│ │ │ rust-version │ 1.60 │ │", + "│ │ │ version │ 0.74.1 │ │", "│ │ │ │ ╭──────────┬────────────────────────╮ │ │", "│ │ │ metadata │ │ │ ╭───────────┬────────╮ │ │ │", - "│ │ │ │ │ binstall │ │ overrides │ {recor │ │ │ │", - "│ │ │ │ │ │ │ │ d 1 │ │ │ │", - "│ │ │ │ │ │ │ │ field} │ │ │ │", - "│ │ │ │ │ │ │ pkg-fmt │ tgz │ │ │ │", - "│ │ │ │ │ │ │ pkg-url │ { repo │ │ │ │", + "│ │ │ │ │ binstall │ │ pkg-url │ { repo │ │ │ │", "│ │ │ │ │ │ │ │ }/rel │ │ │ │", "│ │ │ │ │ │ │ │ eases/ │ │ │ │", "│ │ │ │ │ │ │ │ downlo │ │ │ │", @@ -803,33 +703,14 @@ fn test_expand_big_0() { "│ │ │ │ │ │ │ │ rchive │ │ │ │", "│ │ │ │ │ │ │ │ -forma │ │ │ │", "│ │ │ │ │ │ │ │ t } │ │ │ │", + "│ │ │ │ │ │ │ pkg-fmt │ tgz │ │ │ │", + "│ │ │ │ │ │ │ overrides │ {recor │ │ │ │", + "│ │ │ │ │ │ │ │ d 1 │ │ │ │", + "│ │ │ │ │ │ │ │ field} │ │ │ │", "│ │ │ │ │ │ ╰───────────┴────────╯ │ │ │", "│ │ │ │ ╰──────────┴────────────────────────╯ │ │", - "│ │ │ name │ nu │ │", - "│ │ │ repository │ https://github.com/nushell/nushell │ │", - "│ │ │ rust-version │ 1.60 │ │", - "│ │ │ version │ 0.74.1 │ │", "│ │ ╰───────────────┴───────────────────────────────────────╯ │", "│ │ ╭───────────┬───────────────────────────────────────────╮ │", - "│ patch │ │ │ ╭──────────┬────────────────────────────╮ │ │", - "│ │ │ crates-io │ │ │ ╭────────┬───────────────╮ │ │ │", - "│ │ │ │ │ reedline │ │ branch │ main │ │ │ │", - "│ │ │ │ │ │ │ git │ https://githu │ │ │ │", - "│ │ │ │ │ │ │ │ b.com/nushell │ │ │ │", - "│ │ │ │ │ │ │ │ /reedline.git │ │ │ │", - "│ │ │ │ │ │ ╰────────┴───────────────╯ │ │ │", - "│ │ │ │ ╰──────────┴────────────────────────────╯ │ │", - "│ │ ╰───────────┴───────────────────────────────────────────╯ │", - "│ │ ╭─────────────────────────────────┬─────────────────────╮ │", - "│ target │ │ │ ╭──────────────┬──╮ │ │", - "│ │ │ cfg(not(target_os = \"windows\")) │ │ dependencies │ │ │ │", - "│ │ │ │ ╰──────────────┴──╯ │ │", - "│ │ │ │ ╭──────────────┬──╮ │ │", - "│ │ │ cfg(target_family = \"unix\") │ │ dependencies │ │ │ │", - "│ │ │ │ ╰──────────────┴──╯ │ │", - "│ │ │ cfg(windows) │ {record 1 field} │ │", - "│ │ ╰─────────────────────────────────┴─────────────────────╯ │", - "│ │ ╭───────────┬───────────────────────────────────────────╮ │", "│ workspace │ │ │ ╭────┬────────────────────────────────╮ │ │", "│ │ │ members │ │ 0 │ crates/nu-cli │ │ │", "│ │ │ │ │ 1 │ crates/nu-engine │ │ │", @@ -846,6 +727,125 @@ fn test_expand_big_0() { "│ │ │ │ │ 12 │ crates/nu-utils │ │ │", "│ │ │ │ ╰────┴────────────────────────────────╯ │ │", "│ │ ╰───────────┴───────────────────────────────────────────╯ │", + "│ │ ╭───────────────┬───────────────────────────────────────╮ │", + "│ dependencies │ │ │ ╭──────────┬───────────────╮ │ │", + "│ │ │ chrono │ │ version │ 0.4.23 │ │ │", + "│ │ │ │ │ │ ╭───┬───────╮ │ │ │", + "│ │ │ │ │ features │ │ 0 │ serde │ │ │ │", + "│ │ │ │ │ │ ╰───┴───────╯ │ │ │", + "│ │ │ │ ╰──────────┴───────────────╯ │ │", + "│ │ │ crossterm │ 0.24.0 │ │", + "│ │ │ ctrlc │ 3.2.1 │ │", + "│ │ │ log │ 0.4 │ │", + "│ │ │ │ ╭──────────┬────────────────────────╮ │ │", + "│ │ │ miette │ │ version │ 5.5.0 │ │ │", + "│ │ │ │ │ │ ╭───┬────────────────╮ │ │ │", + "│ │ │ │ │ features │ │ 0 │ fancy-no-backt │ │ │ │", + "│ │ │ │ │ │ │ │ race │ │ │ │", + "│ │ │ │ │ │ ╰───┴────────────────╯ │ │ │", + "│ │ │ │ ╰──────────┴────────────────────────╯ │ │", + "│ │ │ nu-ansi-term │ 0.46.0 │ │", + "│ │ │ │ ╭─────────┬─────────────────╮ │ │", + "│ │ │ nu-cli │ │ path │ ./crates/nu-cli │ │ │", + "│ │ │ │ │ version │ 0.74.1 │ │ │", + "│ │ │ │ ╰─────────┴─────────────────╯ │ │", + "│ │ │ │ ╭────────────┬──────────────────────╮ │ │", + "│ │ │ nu-engine │ │ path │ ./crates/nu-engine │ │ │", + "│ │ │ │ │ version │ 0.74.1 │ │ │", + "│ │ │ │ ╰────────────┴──────────────────────╯ │ │", + "│ │ │ │ ╭─────────────┬─────────────────────╮ │ │", + "│ │ │ reedline │ │ version │ 0.14.0 │ │ │", + "│ │ │ │ │ │ ╭───┬──────────╮ │ │ │", + "│ │ │ │ │ features │ │ 0 │ bashisms │ │ │ │", + "│ │ │ │ │ │ │ 1 │ sqlite │ │ │ │", + "│ │ │ │ │ │ ╰───┴──────────╯ │ │ │", + "│ │ │ │ ╰─────────────┴─────────────────────╯ │ │", + "│ │ │ rayon │ 1.6.1 │ │", + "│ │ │ is_executable │ 1.0.1 │ │", + "│ │ │ simplelog │ 0.12.0 │ │", + "│ │ │ time │ 0.3.12 │ │", + "│ │ ╰───────────────┴───────────────────────────────────────╯ │", + "│ │ ╭─────────────────────────────────┬─────────────────────╮ │", + "│ target │ │ │ ╭──────────────┬──╮ │ │", + "│ │ │ cfg(not(target_os = \"windows\")) │ │ dependencies │ │ │ │", + "│ │ │ │ ╰──────────────┴──╯ │ │", + "│ │ │ cfg(windows) │ {record 1 field} │ │", + "│ │ │ │ ╭──────────────┬──╮ │ │", + "│ │ │ cfg(target_family = \"unix\") │ │ dependencies │ │ │ │", + "│ │ │ │ ╰──────────────┴──╯ │ │", + "│ │ ╰─────────────────────────────────┴─────────────────────╯ │", + "│ │ ╭───────────────────┬───────────────────────────────────╮ │", + "│ dev-dependencies │ │ │ ╭─────────┬─────────────────────╮ │ │", + "│ │ │ nu-test-support │ │ path │ ./crates/nu-test-su │ │ │", + "│ │ │ │ │ │ pport │ │ │", + "│ │ │ │ │ version │ 0.74.1 │ │ │", + "│ │ │ │ ╰─────────┴─────────────────────╯ │ │", + "│ │ │ tempfile │ 3.2.0 │ │", + "│ │ │ assert_cmd │ 2.0.2 │ │", + "│ │ │ criterion │ 0.4 │ │", + "│ │ │ pretty_assertions │ 1.0.0 │ │", + "│ │ │ serial_test │ 0.10.0 │ │", + "│ │ │ hamcrest2 │ 0.3.0 │ │", + "│ │ │ │ ╭────────────────────┬──────────╮ │ │", + "│ │ │ rstest │ │ version │ 0.15.0 │ │ │", + "│ │ │ │ │ default-features │ false │ │ │", + "│ │ │ │ ╰────────────────────┴──────────╯ │ │", + "│ │ │ itertools │ 0.10.3 │ │", + "│ │ ╰───────────────────┴───────────────────────────────────╯ │", + "│ │ ╭─────────────────────┬─────────────────────────────────╮ │", + "│ features │ │ │ ╭───┬────────────────────╮ │ │", + "│ │ │ plugin │ │ 0 │ nu-plugin │ │ │", + "│ │ │ │ │ 1 │ nu-cli/plugin │ │ │", + "│ │ │ │ │ 2 │ nu-parser/plugin │ │ │", + "│ │ │ │ │ 3 │ nu-command/plugin │ │ │", + "│ │ │ │ │ 4 │ nu-protocol/plugin │ │ │", + "│ │ │ │ │ 5 │ nu-engine/plugin │ │ │", + "│ │ │ │ ╰───┴────────────────────╯ │ │", + "│ │ │ │ ╭───┬─────────╮ │ │", + "│ │ │ extra │ │ 0 │ default │ │ │", + "│ │ │ │ ╰───┴─────────╯ │ │", + "│ │ │ │ ╭───┬───────────────╮ │ │", + "│ │ │ default │ │ 0 │ plugin │ │ │", + "│ │ │ │ │ 1 │ which-support │ │ │", + "│ │ │ │ │ 2 │ trash-support │ │ │", + "│ │ │ │ │ 3 │ sqlite │ │ │", + "│ │ │ │ ╰───┴───────────────╯ │ │", + "│ │ │ │ ╭───┬─────────╮ │ │", + "│ │ │ stable │ │ 0 │ default │ │ │", + "│ │ │ │ ╰───┴─────────╯ │ │", + "│ │ │ wasi │ [list 0 items] │ │", + "│ │ │ │ ╭───┬─────────────╮ │ │", + "│ │ │ static-link-openssl │ │ 0 │ dep:openssl │ │ │", + "│ │ │ │ ╰───┴─────────────╯ │ │", + "│ │ │ │ ╭───┬─────────────────────────╮ │ │", + "│ │ │ which-support │ │ 0 │ nu-command/which-suppor │ │ │", + "│ │ │ │ │ │ t │ │ │", + "│ │ │ │ ╰───┴─────────────────────────╯ │ │", + "│ │ │ │ ╭───┬─────────────────────────╮ │ │", + "│ │ │ trash-support │ │ 0 │ nu-command/trash-suppor │ │ │", + "│ │ │ │ │ │ t │ │ │", + "│ │ │ │ ╰───┴─────────────────────────╯ │ │", + "│ │ ╰─────────────────────┴─────────────────────────────────╯ │", + "│ │ ╭───┬──────┬─────────────╮ │", + "│ bin │ │ # │ name │ path │ │", + "│ │ ├───┼──────┼─────────────┤ │", + "│ │ │ 0 │ nu │ src/main.rs │ │", + "│ │ ╰───┴──────┴─────────────╯ │", + "│ │ ╭───────────┬───────────────────────────────────────────╮ │", + "│ patch │ │ │ ╭──────────┬────────────────────────────╮ │ │", + "│ │ │ crates-io │ │ │ ╭────────┬───────────────╮ │ │ │", + "│ │ │ │ │ reedline │ │ git │ https://githu │ │ │ │", + "│ │ │ │ │ │ │ │ b.com/nushell │ │ │ │", + "│ │ │ │ │ │ │ │ /reedline.git │ │ │ │", + "│ │ │ │ │ │ │ branch │ main │ │ │ │", + "│ │ │ │ │ │ ╰────────┴───────────────╯ │ │ │", + "│ │ │ │ ╰──────────┴────────────────────────────╯ │ │", + "│ │ ╰───────────┴───────────────────────────────────────────╯ │", + "│ │ ╭───┬────────────┬─────────╮ │", + "│ bench │ │ # │ name │ harness │ │", + "│ │ ├───┼────────────┼─────────┤ │", + "│ │ │ 0 │ benchmarks │ false │ │", + "│ │ ╰───┴────────────┴─────────╯ │", "╰──────────────────┴───────────────────────────────────────────────────────────╯", ]); @@ -858,102 +858,6 @@ fn test_expand_big_0() { let expected = join_lines([ "╭──────────────────┬───────────────────────────────────────────────────────────────────────────────────────────────────╮", - "│ │ ╭───┬─────────┬────────────╮ │", - "│ bench │ │ # │ harness │ name │ │", - "│ │ ├───┼─────────┼────────────┤ │", - "│ │ │ 0 │ false │ benchmarks │ │", - "│ │ ╰───┴─────────┴────────────╯ │", - "│ │ ╭───┬──────┬─────────────╮ │", - "│ bin │ │ # │ name │ path │ │", - "│ │ ├───┼──────┼─────────────┤ │", - "│ │ │ 0 │ nu │ src/main.rs │ │", - "│ │ ╰───┴──────┴─────────────╯ │", - "│ │ ╭───────────────┬───────────────────────────────────────────╮ │", - "│ dependencies │ │ │ ╭──────────┬───────────────╮ │ │", - "│ │ │ chrono │ │ │ ╭───┬───────╮ │ │ │", - "│ │ │ │ │ features │ │ 0 │ serde │ │ │ │", - "│ │ │ │ │ │ ╰───┴───────╯ │ │ │", - "│ │ │ │ │ version │ 0.4.23 │ │ │", - "│ │ │ │ ╰──────────┴───────────────╯ │ │", - "│ │ │ crossterm │ 0.24.0 │ │", - "│ │ │ ctrlc │ 3.2.1 │ │", - "│ │ │ is_executable │ 1.0.1 │ │", - "│ │ │ log │ 0.4 │ │", - "│ │ │ │ ╭──────────┬────────────────────────────╮ │ │", - "│ │ │ miette │ │ │ ╭───┬────────────────────╮ │ │ │", - "│ │ │ │ │ features │ │ 0 │ fancy-no-backtrace │ │ │ │", - "│ │ │ │ │ │ ╰───┴────────────────────╯ │ │ │", - "│ │ │ │ │ version │ 5.5.0 │ │ │", - "│ │ │ │ ╰──────────┴────────────────────────────╯ │ │", - "│ │ │ nu-ansi-term │ 0.46.0 │ │", - "│ │ │ │ ╭─────────┬─────────────────╮ │ │", - "│ │ │ nu-cli │ │ path │ ./crates/nu-cli │ │ │", - "│ │ │ │ │ version │ 0.74.1 │ │ │", - "│ │ │ │ ╰─────────┴─────────────────╯ │ │", - "│ │ │ │ ╭─────────┬────────────────────╮ │ │", - "│ │ │ nu-engine │ │ path │ ./crates/nu-engine │ │ │", - "│ │ │ │ │ version │ 0.74.1 │ │ │", - "│ │ │ │ ╰─────────┴────────────────────╯ │ │", - "│ │ │ rayon │ 1.6.1 │ │", - "│ │ │ │ ╭──────────┬──────────────────╮ │ │", - "│ │ │ reedline │ │ │ ╭───┬──────────╮ │ │ │", - "│ │ │ │ │ features │ │ 0 │ bashisms │ │ │ │", - "│ │ │ │ │ │ │ 1 │ sqlite │ │ │ │", - "│ │ │ │ │ │ ╰───┴──────────╯ │ │ │", - "│ │ │ │ │ version │ 0.14.0 │ │ │", - "│ │ │ │ ╰──────────┴──────────────────╯ │ │", - "│ │ │ simplelog │ 0.12.0 │ │", - "│ │ │ time │ 0.3.12 │ │", - "│ │ ╰───────────────┴───────────────────────────────────────────╯ │", - "│ │ ╭───────────────────┬────────────────────────────────────────╮ │", - "│ dev-dependencies │ │ assert_cmd │ 2.0.2 │ │", - "│ │ │ criterion │ 0.4 │ │", - "│ │ │ hamcrest2 │ 0.3.0 │ │", - "│ │ │ itertools │ 0.10.3 │ │", - "│ │ │ │ ╭─────────┬──────────────────────────╮ │ │", - "│ │ │ nu-test-support │ │ path │ ./crates/nu-test-support │ │ │", - "│ │ │ │ │ version │ 0.74.1 │ │ │", - "│ │ │ │ ╰─────────┴──────────────────────────╯ │ │", - "│ │ │ pretty_assertions │ 1.0.0 │ │", - "│ │ │ │ ╭──────────────────┬────────╮ │ │", - "│ │ │ rstest │ │ default-features │ false │ │ │", - "│ │ │ │ │ version │ 0.15.0 │ │ │", - "│ │ │ │ ╰──────────────────┴────────╯ │ │", - "│ │ │ serial_test │ 0.10.0 │ │", - "│ │ │ tempfile │ 3.2.0 │ │", - "│ │ ╰───────────────────┴────────────────────────────────────────╯ │", - "│ │ ╭─────────────────────┬──────────────────────────────────╮ │", - "│ features │ │ │ ╭───┬───────────────╮ │ │", - "│ │ │ default │ │ 0 │ plugin │ │ │", - "│ │ │ │ │ 1 │ which-support │ │ │", - "│ │ │ │ │ 2 │ trash-support │ │ │", - "│ │ │ │ │ 3 │ sqlite │ │ │", - "│ │ │ │ ╰───┴───────────────╯ │ │", - "│ │ │ │ ╭───┬─────────╮ │ │", - "│ │ │ extra │ │ 0 │ default │ │ │", - "│ │ │ │ ╰───┴─────────╯ │ │", - "│ │ │ │ ╭───┬────────────────────╮ │ │", - "│ │ │ plugin │ │ 0 │ nu-plugin │ │ │", - "│ │ │ │ │ 1 │ nu-cli/plugin │ │ │", - "│ │ │ │ │ 2 │ nu-parser/plugin │ │ │", - "│ │ │ │ │ 3 │ nu-command/plugin │ │ │", - "│ │ │ │ │ 4 │ nu-protocol/plugin │ │ │", - "│ │ │ │ │ 5 │ nu-engine/plugin │ │ │", - "│ │ │ │ ╰───┴────────────────────╯ │ │", - "│ │ │ │ ╭───┬─────────╮ │ │", - "│ │ │ stable │ │ 0 │ default │ │ │", - "│ │ │ │ ╰───┴─────────╯ │ │", - "│ │ │ │ ╭───┬─────────────╮ │ │", - "│ │ │ static-link-openssl │ │ 0 │ dep:openssl │ │ │", - "│ │ │ │ ╰───┴─────────────╯ │ │", - "│ │ │ │ ╭───┬──────────────────────────╮ │ │", - "│ │ │ trash-support │ │ 0 │ nu-command/trash-support │ │ │", - "│ │ │ │ ╰───┴──────────────────────────╯ │ │", - "│ │ │ wasi │ [list 0 items] │ │", - "│ │ │ │ ╭───┬──────────────────────────╮ │ │", - "│ │ │ which-support │ │ 0 │ nu-command/which-support │ │ │", - "│ │ │ │ ╰───┴──────────────────────────╯ │ │", - "│ │ ╰─────────────────────┴──────────────────────────────────╯ │", "│ │ ╭───────────────┬───────────────────────────────────────────────────────────────────────────────╮ │", "│ package │ │ │ ╭───┬────────────────────────────────╮ │ │", "│ │ │ authors │ │ 0 │ The Nushell Project Developers │ │ │", @@ -967,61 +871,23 @@ fn test_expand_big_0() { "│ │ │ │ ╰───┴────────╯ │ │", "│ │ │ homepage │ https://www.nushell.sh │ │", "│ │ │ license │ MIT │ │", - "│ │ │ │ ╭──────────┬────────────────────────────────────────────────────────────────╮ │ │", - "│ │ │ metadata │ │ │ ╭───────────┬────────────────────────────────────────────────╮ │ │ │", - "│ │ │ │ │ binstall │ │ │ ╭────────────────────────┬───────────────────╮ │ │ │ │", - "│ │ │ │ │ │ │ overrides │ │ │ ╭─────────┬─────╮ │ │ │ │ │", - "│ │ │ │ │ │ │ │ │ x86_64-pc-windows-msvc │ │ pkg-fmt │ zip │ │ │ │ │ │", - "│ │ │ │ │ │ │ │ │ │ ╰─────────┴─────╯ │ │ │ │ │", - "│ │ │ │ │ │ │ │ ╰────────────────────────┴───────────────────╯ │ │ │ │", - "│ │ │ │ │ │ │ pkg-fmt │ tgz │ │ │ │", - "│ │ │ │ │ │ │ pkg-url │ { repo }/releases/download/{ version }/{ name │ │ │ │", - "│ │ │ │ │ │ │ │ }-{ version }-{ target }.{ archive-format } │ │ │ │", - "│ │ │ │ │ │ ╰───────────┴────────────────────────────────────────────────╯ │ │ │", - "│ │ │ │ ╰──────────┴────────────────────────────────────────────────────────────────╯ │ │", "│ │ │ name │ nu │ │", "│ │ │ repository │ https://github.com/nushell/nushell │ │", "│ │ │ rust-version │ 1.60 │ │", "│ │ │ version │ 0.74.1 │ │", + "│ │ │ │ ╭──────────┬────────────────────────────────────────────────────────────────╮ │ │", + "│ │ │ metadata │ │ │ ╭───────────┬────────────────────────────────────────────────╮ │ │ │", + "│ │ │ │ │ binstall │ │ pkg-url │ { repo }/releases/download/{ version }/{ name │ │ │ │", + "│ │ │ │ │ │ │ │ }-{ version }-{ target }.{ archive-format } │ │ │ │", + "│ │ │ │ │ │ │ pkg-fmt │ tgz │ │ │ │", + "│ │ │ │ │ │ │ │ ╭────────────────────────┬───────────────────╮ │ │ │ │", + "│ │ │ │ │ │ │ overrides │ │ │ ╭─────────┬─────╮ │ │ │ │ │", + "│ │ │ │ │ │ │ │ │ x86_64-pc-windows-msvc │ │ pkg-fmt │ zip │ │ │ │ │ │", + "│ │ │ │ │ │ │ │ │ │ ╰─────────┴─────╯ │ │ │ │ │", + "│ │ │ │ │ │ │ │ ╰────────────────────────┴───────────────────╯ │ │ │ │", + "│ │ │ │ │ │ ╰───────────┴────────────────────────────────────────────────╯ │ │ │", + "│ │ │ │ ╰──────────┴────────────────────────────────────────────────────────────────╯ │ │", "│ │ ╰───────────────┴───────────────────────────────────────────────────────────────────────────────╯ │", - "│ │ ╭───────────┬───────────────────────────────────────────────────────────────────────────────────╮ │", - "│ patch │ │ │ ╭─────────────────┬─────────────────────────────────────────────────────────────╮ │ │", - "│ │ │ crates-io │ │ │ ╭────────┬─────────────────────────────────────────╮ │ │ │", - "│ │ │ │ │ reedline │ │ branch │ main │ │ │ │", - "│ │ │ │ │ │ │ git │ https://github.com/nushell/reedline.git │ │ │ │", - "│ │ │ │ │ │ ╰────────┴─────────────────────────────────────────╯ │ │ │", - "│ │ │ │ ╰─────────────────┴─────────────────────────────────────────────────────────────╯ │ │", - "│ │ ╰───────────┴───────────────────────────────────────────────────────────────────────────────────╯ │", - "│ │ ╭─────────────────────────────────┬─────────────────────────────────────────────────────────────╮ │", - "│ target │ │ │ ╭──────────────┬──────────────────────────────────────────╮ │ │", - "│ │ │ cfg(not(target_os = \"windows\")) │ │ │ ╭─────────────┬────────────────────────╮ │ │ │", - "│ │ │ │ │ dependencies │ │ │ ╭──────────┬─────────╮ │ │ │ │", - "│ │ │ │ │ │ │ openssl │ │ features │ [list 1 │ │ │ │ │", - "│ │ │ │ │ │ │ │ │ │ item] │ │ │ │ │", - "│ │ │ │ │ │ │ │ │ optional │ true │ │ │ │ │", - "│ │ │ │ │ │ │ │ │ version │ 0.10.38 │ │ │ │ │", - "│ │ │ │ │ │ │ │ ╰──────────┴─────────╯ │ │ │ │", - "│ │ │ │ │ │ │ signal-hook │ {record 2 fields} │ │ │ │", - "│ │ │ │ │ │ ╰─────────────┴────────────────────────╯ │ │ │", - "│ │ │ │ ╰──────────────┴──────────────────────────────────────────╯ │ │", - "│ │ │ │ ╭──────────────┬──────────────────────────────────────────╮ │ │", - "│ │ │ cfg(target_family = \"unix\") │ │ │ ╭──────┬───────────────────────────────╮ │ │ │", - "│ │ │ │ │ dependencies │ │ atty │ 0.2 │ │ │ │", - "│ │ │ │ │ │ │ │ ╭──────────────────┬────────╮ │ │ │ │", - "│ │ │ │ │ │ │ nix │ │ default-features │ false │ │ │ │ │", - "│ │ │ │ │ │ │ │ │ features │ [list │ │ │ │ │", - "│ │ │ │ │ │ │ │ │ │ 4 │ │ │ │ │", - "│ │ │ │ │ │ │ │ │ │ items] │ │ │ │ │", - "│ │ │ │ │ │ │ │ │ version │ 0.25 │ │ │ │ │", - "│ │ │ │ │ │ │ │ ╰──────────────────┴────────╯ │ │ │ │", - "│ │ │ │ │ │ ╰──────┴───────────────────────────────╯ │ │ │", - "│ │ │ │ ╰──────────────┴──────────────────────────────────────────╯ │ │", - "│ │ │ │ ╭────────────────────┬──────────────────╮ │ │", - "│ │ │ cfg(windows) │ │ │ ╭────────┬─────╮ │ │ │", - "│ │ │ │ │ build-dependencies │ │ winres │ 0.1 │ │ │ │", - "│ │ │ │ │ │ ╰────────┴─────╯ │ │ │", - "│ │ │ │ ╰────────────────────┴──────────────────╯ │ │", - "│ │ ╰─────────────────────────────────┴─────────────────────────────────────────────────────────────╯ │", "│ │ ╭─────────┬─────────────────────────────────────────╮ │", "│ workspace │ │ │ ╭────┬────────────────────────────────╮ │ │", "│ │ │ members │ │ 0 │ crates/nu-cli │ │ │", @@ -1039,6 +905,140 @@ fn test_expand_big_0() { "│ │ │ │ │ 12 │ crates/nu-utils │ │ │", "│ │ │ │ ╰────┴────────────────────────────────╯ │ │", "│ │ ╰─────────┴─────────────────────────────────────────╯ │", + "│ │ ╭───────────────┬───────────────────────────────────────────╮ │", + "│ dependencies │ │ │ ╭──────────┬───────────────╮ │ │", + "│ │ │ chrono │ │ version │ 0.4.23 │ │ │", + "│ │ │ │ │ │ ╭───┬───────╮ │ │ │", + "│ │ │ │ │ features │ │ 0 │ serde │ │ │ │", + "│ │ │ │ │ │ ╰───┴───────╯ │ │ │", + "│ │ │ │ ╰──────────┴───────────────╯ │ │", + "│ │ │ crossterm │ 0.24.0 │ │", + "│ │ │ ctrlc │ 3.2.1 │ │", + "│ │ │ log │ 0.4 │ │", + "│ │ │ │ ╭──────────┬────────────────────────────╮ │ │", + "│ │ │ miette │ │ version │ 5.5.0 │ │ │", + "│ │ │ │ │ │ ╭───┬────────────────────╮ │ │ │", + "│ │ │ │ │ features │ │ 0 │ fancy-no-backtrace │ │ │ │", + "│ │ │ │ │ │ ╰───┴────────────────────╯ │ │ │", + "│ │ │ │ ╰──────────┴────────────────────────────╯ │ │", + "│ │ │ nu-ansi-term │ 0.46.0 │ │", + "│ │ │ │ ╭─────────┬─────────────────╮ │ │", + "│ │ │ nu-cli │ │ path │ ./crates/nu-cli │ │ │", + "│ │ │ │ │ version │ 0.74.1 │ │ │", + "│ │ │ │ ╰─────────┴─────────────────╯ │ │", + "│ │ │ │ ╭─────────┬────────────────────╮ │ │", + "│ │ │ nu-engine │ │ path │ ./crates/nu-engine │ │ │", + "│ │ │ │ │ version │ 0.74.1 │ │ │", + "│ │ │ │ ╰─────────┴────────────────────╯ │ │", + "│ │ │ │ ╭──────────┬──────────────────╮ │ │", + "│ │ │ reedline │ │ version │ 0.14.0 │ │ │", + "│ │ │ │ │ │ ╭───┬──────────╮ │ │ │", + "│ │ │ │ │ features │ │ 0 │ bashisms │ │ │ │", + "│ │ │ │ │ │ │ 1 │ sqlite │ │ │ │", + "│ │ │ │ │ │ ╰───┴──────────╯ │ │ │", + "│ │ │ │ ╰──────────┴──────────────────╯ │ │", + "│ │ │ rayon │ 1.6.1 │ │", + "│ │ │ is_executable │ 1.0.1 │ │", + "│ │ │ simplelog │ 0.12.0 │ │", + "│ │ │ time │ 0.3.12 │ │", + "│ │ ╰───────────────┴───────────────────────────────────────────╯ │", + "│ │ ╭─────────────────────────────────┬─────────────────────────────────────────────────────────────╮ │", + "│ target │ │ │ ╭──────────────┬──────────────────────────────────────────╮ │ │", + "│ │ │ cfg(not(target_os = \"windows\")) │ │ │ ╭─────────────┬────────────────────────╮ │ │ │", + "│ │ │ │ │ dependencies │ │ │ ╭──────────┬─────────╮ │ │ │ │", + "│ │ │ │ │ │ │ openssl │ │ version │ 0.10.38 │ │ │ │ │", + "│ │ │ │ │ │ │ │ │ features │ [list 1 │ │ │ │ │", + "│ │ │ │ │ │ │ │ │ │ item] │ │ │ │ │", + "│ │ │ │ │ │ │ │ │ optional │ true │ │ │ │ │", + "│ │ │ │ │ │ │ │ ╰──────────┴─────────╯ │ │ │ │", + "│ │ │ │ │ │ │ signal-hook │ {record 2 fields} │ │ │ │", + "│ │ │ │ │ │ ╰─────────────┴────────────────────────╯ │ │ │", + "│ │ │ │ ╰──────────────┴──────────────────────────────────────────╯ │ │", + "│ │ │ │ ╭────────────────────┬──────────────────╮ │ │", + "│ │ │ cfg(windows) │ │ │ ╭────────┬─────╮ │ │ │", + "│ │ │ │ │ build-dependencies │ │ winres │ 0.1 │ │ │ │", + "│ │ │ │ │ │ ╰────────┴─────╯ │ │ │", + "│ │ │ │ ╰────────────────────┴──────────────────╯ │ │", + "│ │ │ │ ╭──────────────┬──────────────────────────────────────────╮ │ │", + "│ │ │ cfg(target_family = \"unix\") │ │ │ ╭──────┬───────────────────────────────╮ │ │ │", + "│ │ │ │ │ dependencies │ │ │ ╭──────────────────┬────────╮ │ │ │ │", + "│ │ │ │ │ │ │ nix │ │ version │ 0.25 │ │ │ │ │", + "│ │ │ │ │ │ │ │ │ default-features │ false │ │ │ │ │", + "│ │ │ │ │ │ │ │ │ features │ [list │ │ │ │ │", + "│ │ │ │ │ │ │ │ │ │ 4 │ │ │ │ │", + "│ │ │ │ │ │ │ │ │ │ items] │ │ │ │ │", + "│ │ │ │ │ │ │ │ ╰──────────────────┴────────╯ │ │ │ │", + "│ │ │ │ │ │ │ atty │ 0.2 │ │ │ │", + "│ │ │ │ │ │ ╰──────┴───────────────────────────────╯ │ │ │", + "│ │ │ │ ╰──────────────┴──────────────────────────────────────────╯ │ │", + "│ │ ╰─────────────────────────────────┴─────────────────────────────────────────────────────────────╯ │", + "│ │ ╭───────────────────┬────────────────────────────────────────╮ │", + "│ dev-dependencies │ │ │ ╭─────────┬──────────────────────────╮ │ │", + "│ │ │ nu-test-support │ │ path │ ./crates/nu-test-support │ │ │", + "│ │ │ │ │ version │ 0.74.1 │ │ │", + "│ │ │ │ ╰─────────┴──────────────────────────╯ │ │", + "│ │ │ tempfile │ 3.2.0 │ │", + "│ │ │ assert_cmd │ 2.0.2 │ │", + "│ │ │ criterion │ 0.4 │ │", + "│ │ │ pretty_assertions │ 1.0.0 │ │", + "│ │ │ serial_test │ 0.10.0 │ │", + "│ │ │ hamcrest2 │ 0.3.0 │ │", + "│ │ │ │ ╭──────────────────┬────────╮ │ │", + "│ │ │ rstest │ │ version │ 0.15.0 │ │ │", + "│ │ │ │ │ default-features │ false │ │ │", + "│ │ │ │ ╰──────────────────┴────────╯ │ │", + "│ │ │ itertools │ 0.10.3 │ │", + "│ │ ╰───────────────────┴────────────────────────────────────────╯ │", + "│ │ ╭─────────────────────┬──────────────────────────────────╮ │", + "│ features │ │ │ ╭───┬────────────────────╮ │ │", + "│ │ │ plugin │ │ 0 │ nu-plugin │ │ │", + "│ │ │ │ │ 1 │ nu-cli/plugin │ │ │", + "│ │ │ │ │ 2 │ nu-parser/plugin │ │ │", + "│ │ │ │ │ 3 │ nu-command/plugin │ │ │", + "│ │ │ │ │ 4 │ nu-protocol/plugin │ │ │", + "│ │ │ │ │ 5 │ nu-engine/plugin │ │ │", + "│ │ │ │ ╰───┴────────────────────╯ │ │", + "│ │ │ │ ╭───┬─────────╮ │ │", + "│ │ │ extra │ │ 0 │ default │ │ │", + "│ │ │ │ ╰───┴─────────╯ │ │", + "│ │ │ │ ╭───┬───────────────╮ │ │", + "│ │ │ default │ │ 0 │ plugin │ │ │", + "│ │ │ │ │ 1 │ which-support │ │ │", + "│ │ │ │ │ 2 │ trash-support │ │ │", + "│ │ │ │ │ 3 │ sqlite │ │ │", + "│ │ │ │ ╰───┴───────────────╯ │ │", + "│ │ │ │ ╭───┬─────────╮ │ │", + "│ │ │ stable │ │ 0 │ default │ │ │", + "│ │ │ │ ╰───┴─────────╯ │ │", + "│ │ │ wasi │ [list 0 items] │ │", + "│ │ │ │ ╭───┬─────────────╮ │ │", + "│ │ │ static-link-openssl │ │ 0 │ dep:openssl │ │ │", + "│ │ │ │ ╰───┴─────────────╯ │ │", + "│ │ │ │ ╭───┬──────────────────────────╮ │ │", + "│ │ │ which-support │ │ 0 │ nu-command/which-support │ │ │", + "│ │ │ │ ╰───┴──────────────────────────╯ │ │", + "│ │ │ │ ╭───┬──────────────────────────╮ │ │", + "│ │ │ trash-support │ │ 0 │ nu-command/trash-support │ │ │", + "│ │ │ │ ╰───┴──────────────────────────╯ │ │", + "│ │ ╰─────────────────────┴──────────────────────────────────╯ │", + "│ │ ╭───┬──────┬─────────────╮ │", + "│ bin │ │ # │ name │ path │ │", + "│ │ ├───┼──────┼─────────────┤ │", + "│ │ │ 0 │ nu │ src/main.rs │ │", + "│ │ ╰───┴──────┴─────────────╯ │", + "│ │ ╭───────────┬───────────────────────────────────────────────────────────────────────────────────╮ │", + "│ patch │ │ │ ╭─────────────────┬─────────────────────────────────────────────────────────────╮ │ │", + "│ │ │ crates-io │ │ │ ╭────────┬─────────────────────────────────────────╮ │ │ │", + "│ │ │ │ │ reedline │ │ git │ https://github.com/nushell/reedline.git │ │ │ │", + "│ │ │ │ │ │ │ branch │ main │ │ │ │", + "│ │ │ │ │ │ ╰────────┴─────────────────────────────────────────╯ │ │ │", + "│ │ │ │ ╰─────────────────┴─────────────────────────────────────────────────────────────╯ │ │", + "│ │ ╰───────────┴───────────────────────────────────────────────────────────────────────────────────╯ │", + "│ │ ╭───┬────────────┬─────────╮ │", + "│ bench │ │ # │ name │ harness │ │", + "│ │ ├───┼────────────┼─────────┤ │", + "│ │ │ 0 │ benchmarks │ false │ │", + "│ │ ╰───┴────────────┴─────────╯ │", "╰──────────────────┴───────────────────────────────────────────────────────────────────────────────────────────────────╯", ]); @@ -1053,44 +1053,99 @@ fn test_expand_big_0() { let expected = join_lines([ "╭──────────────────┬───────────────────────────────────────╮", - "│ │ ╭───┬─────────┬────────────╮ │", - "│ bench │ │ # │ harness │ name │ │", - "│ │ ├───┼─────────┼────────────┤ │", - "│ │ │ 0 │ false │ benchmarks │ │", - "│ │ ╰───┴─────────┴────────────╯ │", - "│ │ ╭───┬──────┬─────────────╮ │", - "│ bin │ │ # │ name │ path │ │", - "│ │ ├───┼──────┼─────────────┤ │", - "│ │ │ 0 │ nu │ src/main.rs │ │", - "│ │ ╰───┴──────┴─────────────╯ │", + "│ │ ╭───────────────┬───────────────────╮ │", + "│ package │ │ │ ╭───┬───────────╮ │ │", + "│ │ │ authors │ │ 0 │ The │ │ │", + "│ │ │ │ │ │ Nushell │ │ │", + "│ │ │ │ │ │ Project D │ │ │", + "│ │ │ │ │ │ evelopers │ │ │", + "│ │ │ │ ╰───┴───────────╯ │ │", + "│ │ │ default-run │ nu │ │", + "│ │ │ description │ A new type of │ │", + "│ │ │ │ shell │ │", + "│ │ │ documentation │ https://www.nushe │ │", + "│ │ │ │ ll.sh/book/ │ │", + "│ │ │ edition │ 2021 │ │", + "│ │ │ │ ╭───┬────────╮ │ │", + "│ │ │ exclude │ │ 0 │ images │ │ │", + "│ │ │ │ ╰───┴────────╯ │ │", + "│ │ │ homepage │ https://www.nushe │ │", + "│ │ │ │ ll.sh │ │", + "│ │ │ license │ MIT │ │", + "│ │ │ name │ nu │ │", + "│ │ │ repository │ https://github.co │ │", + "│ │ │ │ m/nushell/nushell │ │", + "│ │ │ rust-version │ 1.60 │ │", + "│ │ │ version │ 0.74.1 │ │", + "│ │ │ │ ╭──────────┬────╮ │ │", + "│ │ │ metadata │ │ binstall │ {r │ │ │", + "│ │ │ │ │ │ ec │ │ │", + "│ │ │ │ │ │ or │ │ │", + "│ │ │ │ │ │ d │ │ │", + "│ │ │ │ │ │ 3 │ │ │", + "│ │ │ │ │ │ fi │ │ │", + "│ │ │ │ │ │ el │ │ │", + "│ │ │ │ │ │ ds │ │ │", + "│ │ │ │ │ │ } │ │ │", + "│ │ │ │ ╰──────────┴────╯ │ │", + "│ │ ╰───────────────┴───────────────────╯ │", + "│ │ ╭─────────┬─────────────────────────╮ │", + "│ workspace │ │ │ ╭────┬────────────────╮ │ │", + "│ │ │ members │ │ 0 │ crates/nu-cli │ │ │", + "│ │ │ │ │ 1 │ crates/nu-engi │ │ │", + "│ │ │ │ │ │ ne │ │ │", + "│ │ │ │ │ 2 │ crates/nu-pars │ │ │", + "│ │ │ │ │ │ er │ │ │", + "│ │ │ │ │ 3 │ crates/nu-syst │ │ │", + "│ │ │ │ │ │ em │ │ │", + "│ │ │ │ │ 4 │ crates/nu-comm │ │ │", + "│ │ │ │ │ │ and │ │ │", + "│ │ │ │ │ 5 │ crates/nu-prot │ │ │", + "│ │ │ │ │ │ ocol │ │ │", + "│ │ │ │ │ 6 │ crates/nu-plug │ │ │", + "│ │ │ │ │ │ in │ │ │", + "│ │ │ │ │ 7 │ crates/nu_plug │ │ │", + "│ │ │ │ │ │ in_inc │ │ │", + "│ │ │ │ │ 8 │ crates/nu_plug │ │ │", + "│ │ │ │ │ │ in_gstat │ │ │", + "│ │ │ │ │ 9 │ crates/nu_plug │ │ │", + "│ │ │ │ │ │ in_example │ │ │", + "│ │ │ │ │ 10 │ crates/nu_plug │ │ │", + "│ │ │ │ │ │ in_query │ │ │", + "│ │ │ │ │ 11 │ crates/nu_plug │ │ │", + "│ │ │ │ │ │ in_custom_valu │ │ │", + "│ │ │ │ │ │ es │ │ │", + "│ │ │ │ │ 12 │ crates/nu-util │ │ │", + "│ │ │ │ │ │ s │ │ │", + "│ │ │ │ ╰────┴────────────────╯ │ │", + "│ │ ╰─────────┴─────────────────────────╯ │", "│ │ ╭───────────────┬───────────────────╮ │", "│ dependencies │ │ │ ╭──────────┬────╮ │ │", - "│ │ │ chrono │ │ features │ [l │ │ │", + "│ │ │ chrono │ │ version │ 0. │ │ │", + "│ │ │ │ │ │ 4. │ │ │", + "│ │ │ │ │ │ 23 │ │ │", + "│ │ │ │ │ features │ [l │ │ │", "│ │ │ │ │ │ is │ │ │", "│ │ │ │ │ │ t │ │ │", "│ │ │ │ │ │ 1 │ │ │", "│ │ │ │ │ │ it │ │ │", "│ │ │ │ │ │ em │ │ │", "│ │ │ │ │ │ ] │ │ │", - "│ │ │ │ │ version │ 0. │ │ │", - "│ │ │ │ │ │ 4. │ │ │", - "│ │ │ │ │ │ 23 │ │ │", "│ │ │ │ ╰──────────┴────╯ │ │", "│ │ │ crossterm │ 0.24.0 │ │", "│ │ │ ctrlc │ 3.2.1 │ │", - "│ │ │ is_executable │ 1.0.1 │ │", "│ │ │ log │ 0.4 │ │", "│ │ │ │ ╭──────────┬────╮ │ │", - "│ │ │ miette │ │ features │ [l │ │ │", + "│ │ │ miette │ │ version │ 5. │ │ │", + "│ │ │ │ │ │ 5. │ │ │", + "│ │ │ │ │ │ 0 │ │ │", + "│ │ │ │ │ features │ [l │ │ │", "│ │ │ │ │ │ is │ │ │", "│ │ │ │ │ │ t │ │ │", "│ │ │ │ │ │ 1 │ │ │", "│ │ │ │ │ │ it │ │ │", "│ │ │ │ │ │ em │ │ │", "│ │ │ │ │ │ ] │ │ │", - "│ │ │ │ │ version │ 5. │ │ │", - "│ │ │ │ │ │ 5. │ │ │", - "│ │ │ │ │ │ 0 │ │ │", "│ │ │ │ ╰──────────┴────╯ │ │", "│ │ │ nu-ansi-term │ 0.46.0 │ │", "│ │ │ │ ╭─────────┬─────╮ │ │", @@ -1112,58 +1167,39 @@ fn test_expand_big_0() { "│ │ │ │ │ version │ 0.7 │ │ │", "│ │ │ │ │ │ 4.1 │ │ │", "│ │ │ │ ╰─────────┴─────╯ │ │", - "│ │ │ rayon │ 1.6.1 │ │", "│ │ │ │ ╭──────────┬────╮ │ │", - "│ │ │ reedline │ │ features │ [l │ │ │", + "│ │ │ reedline │ │ version │ 0. │ │ │", + "│ │ │ │ │ │ 14 │ │ │", + "│ │ │ │ │ │ .0 │ │ │", + "│ │ │ │ │ features │ [l │ │ │", "│ │ │ │ │ │ is │ │ │", "│ │ │ │ │ │ t │ │ │", "│ │ │ │ │ │ 2 │ │ │", "│ │ │ │ │ │ it │ │ │", "│ │ │ │ │ │ em │ │ │", "│ │ │ │ │ │ s] │ │ │", - "│ │ │ │ │ version │ 0. │ │ │", - "│ │ │ │ │ │ 14 │ │ │", - "│ │ │ │ │ │ .0 │ │ │", "│ │ │ │ ╰──────────┴────╯ │ │", + "│ │ │ rayon │ 1.6.1 │ │", + "│ │ │ is_executable │ 1.0.1 │ │", "│ │ │ simplelog │ 0.12.0 │ │", "│ │ │ time │ 0.3.12 │ │", "│ │ ╰───────────────┴───────────────────╯ │", + "│ target │ {record 3 fields} │", "│ │ ╭───────────────────┬───────────────╮ │", - "│ dev-dependencies │ │ assert_cmd │ 2.0.2 │ │", - "│ │ │ criterion │ 0.4 │ │", - "│ │ │ hamcrest2 │ 0.3.0 │ │", - "│ │ │ itertools │ 0.10.3 │ │", - "│ │ │ nu-test-support │ {record 2 │ │", + "│ dev-dependencies │ │ nu-test-support │ {record 2 │ │", "│ │ │ │ fields} │ │", + "│ │ │ tempfile │ 3.2.0 │ │", + "│ │ │ assert_cmd │ 2.0.2 │ │", + "│ │ │ criterion │ 0.4 │ │", "│ │ │ pretty_assertions │ 1.0.0 │ │", + "│ │ │ serial_test │ 0.10.0 │ │", + "│ │ │ hamcrest2 │ 0.3.0 │ │", "│ │ │ rstest │ {record 2 │ │", "│ │ │ │ fields} │ │", - "│ │ │ serial_test │ 0.10.0 │ │", - "│ │ │ tempfile │ 3.2.0 │ │", + "│ │ │ itertools │ 0.10.3 │ │", "│ │ ╰───────────────────┴───────────────╯ │", "│ │ ╭─────────────────────┬─────────────╮ │", "│ features │ │ │ ╭───┬─────╮ │ │", - "│ │ │ default │ │ 0 │ plu │ │ │", - "│ │ │ │ │ │ gin │ │ │", - "│ │ │ │ │ 1 │ whi │ │ │", - "│ │ │ │ │ │ ch- │ │ │", - "│ │ │ │ │ │ sup │ │ │", - "│ │ │ │ │ │ por │ │ │", - "│ │ │ │ │ │ t │ │ │", - "│ │ │ │ │ 2 │ tra │ │ │", - "│ │ │ │ │ │ sh- │ │ │", - "│ │ │ │ │ │ sup │ │ │", - "│ │ │ │ │ │ por │ │ │", - "│ │ │ │ │ │ t │ │ │", - "│ │ │ │ │ 3 │ sql │ │ │", - "│ │ │ │ │ │ ite │ │ │", - "│ │ │ │ ╰───┴─────╯ │ │", - "│ │ │ │ ╭───┬─────╮ │ │", - "│ │ │ extra │ │ 0 │ def │ │ │", - "│ │ │ │ │ │ aul │ │ │", - "│ │ │ │ │ │ t │ │ │", - "│ │ │ │ ╰───┴─────╯ │ │", - "│ │ │ │ ╭───┬─────╮ │ │", "│ │ │ plugin │ │ 0 │ nu- │ │ │", "│ │ │ │ │ │ plu │ │ │", "│ │ │ │ │ │ gin │ │ │", @@ -1198,10 +1234,33 @@ fn test_expand_big_0() { "│ │ │ │ │ │ n │ │ │", "│ │ │ │ ╰───┴─────╯ │ │", "│ │ │ │ ╭───┬─────╮ │ │", + "│ │ │ extra │ │ 0 │ def │ │ │", + "│ │ │ │ │ │ aul │ │ │", + "│ │ │ │ │ │ t │ │ │", + "│ │ │ │ ╰───┴─────╯ │ │", + "│ │ │ │ ╭───┬─────╮ │ │", + "│ │ │ default │ │ 0 │ plu │ │ │", + "│ │ │ │ │ │ gin │ │ │", + "│ │ │ │ │ 1 │ whi │ │ │", + "│ │ │ │ │ │ ch- │ │ │", + "│ │ │ │ │ │ sup │ │ │", + "│ │ │ │ │ │ por │ │ │", + "│ │ │ │ │ │ t │ │ │", + "│ │ │ │ │ 2 │ tra │ │ │", + "│ │ │ │ │ │ sh- │ │ │", + "│ │ │ │ │ │ sup │ │ │", + "│ │ │ │ │ │ por │ │ │", + "│ │ │ │ │ │ t │ │ │", + "│ │ │ │ │ 3 │ sql │ │ │", + "│ │ │ │ │ │ ite │ │ │", + "│ │ │ │ ╰───┴─────╯ │ │", + "│ │ │ │ ╭───┬─────╮ │ │", "│ │ │ stable │ │ 0 │ def │ │ │", "│ │ │ │ │ │ aul │ │ │", "│ │ │ │ │ │ t │ │ │", "│ │ │ │ ╰───┴─────╯ │ │", + "│ │ │ wasi │ [list 0 │ │", + "│ │ │ │ items] │ │", "│ │ │ │ ╭───┬─────╮ │ │", "│ │ │ static-link-openssl │ │ 0 │ dep │ │ │", "│ │ │ │ │ │ :op │ │ │", @@ -1209,6 +1268,16 @@ fn test_expand_big_0() { "│ │ │ │ │ │ sl │ │ │", "│ │ │ │ ╰───┴─────╯ │ │", "│ │ │ │ ╭───┬─────╮ │ │", + "│ │ │ which-support │ │ 0 │ nu- │ │ │", + "│ │ │ │ │ │ com │ │ │", + "│ │ │ │ │ │ man │ │ │", + "│ │ │ │ │ │ d/w │ │ │", + "│ │ │ │ │ │ hic │ │ │", + "│ │ │ │ │ │ h-s │ │ │", + "│ │ │ │ │ │ upp │ │ │", + "│ │ │ │ │ │ ort │ │ │", + "│ │ │ │ ╰───┴─────╯ │ │", + "│ │ │ │ ╭───┬─────╮ │ │", "│ │ │ trash-support │ │ 0 │ nu- │ │ │", "│ │ │ │ │ │ com │ │ │", "│ │ │ │ │ │ man │ │ │", @@ -1218,55 +1287,12 @@ fn test_expand_big_0() { "│ │ │ │ │ │ upp │ │ │", "│ │ │ │ │ │ ort │ │ │", "│ │ │ │ ╰───┴─────╯ │ │", - "│ │ │ wasi │ [list 0 │ │", - "│ │ │ │ items] │ │", - "│ │ │ │ ╭───┬─────╮ │ │", - "│ │ │ which-support │ │ 0 │ nu- │ │ │", - "│ │ │ │ │ │ com │ │ │", - "│ │ │ │ │ │ man │ │ │", - "│ │ │ │ │ │ d/w │ │ │", - "│ │ │ │ │ │ hic │ │ │", - "│ │ │ │ │ │ h-s │ │ │", - "│ │ │ │ │ │ upp │ │ │", - "│ │ │ │ │ │ ort │ │ │", - "│ │ │ │ ╰───┴─────╯ │ │", "│ │ ╰─────────────────────┴─────────────╯ │", - "│ │ ╭───────────────┬───────────────────╮ │", - "│ package │ │ │ ╭───┬───────────╮ │ │", - "│ │ │ authors │ │ 0 │ The │ │ │", - "│ │ │ │ │ │ Nushell │ │ │", - "│ │ │ │ │ │ Project D │ │ │", - "│ │ │ │ │ │ evelopers │ │ │", - "│ │ │ │ ╰───┴───────────╯ │ │", - "│ │ │ default-run │ nu │ │", - "│ │ │ description │ A new type of │ │", - "│ │ │ │ shell │ │", - "│ │ │ documentation │ https://www.nushe │ │", - "│ │ │ │ ll.sh/book/ │ │", - "│ │ │ edition │ 2021 │ │", - "│ │ │ │ ╭───┬────────╮ │ │", - "│ │ │ exclude │ │ 0 │ images │ │ │", - "│ │ │ │ ╰───┴────────╯ │ │", - "│ │ │ homepage │ https://www.nushe │ │", - "│ │ │ │ ll.sh │ │", - "│ │ │ license │ MIT │ │", - "│ │ │ │ ╭──────────┬────╮ │ │", - "│ │ │ metadata │ │ binstall │ {r │ │ │", - "│ │ │ │ │ │ ec │ │ │", - "│ │ │ │ │ │ or │ │ │", - "│ │ │ │ │ │ d │ │ │", - "│ │ │ │ │ │ 3 │ │ │", - "│ │ │ │ │ │ fi │ │ │", - "│ │ │ │ │ │ el │ │ │", - "│ │ │ │ │ │ ds │ │ │", - "│ │ │ │ │ │ } │ │ │", - "│ │ │ │ ╰──────────┴────╯ │ │", - "│ │ │ name │ nu │ │", - "│ │ │ repository │ https://github.co │ │", - "│ │ │ │ m/nushell/nushell │ │", - "│ │ │ rust-version │ 1.60 │ │", - "│ │ │ version │ 0.74.1 │ │", - "│ │ ╰───────────────┴───────────────────╯ │", + "│ │ ╭───┬──────┬─────────────╮ │", + "│ bin │ │ # │ name │ path │ │", + "│ │ ├───┼──────┼─────────────┤ │", + "│ │ │ 0 │ nu │ src/main.rs │ │", + "│ │ ╰───┴──────┴─────────────╯ │", "│ │ ╭───────────┬───────────────────────╮ │", "│ patch │ │ │ ╭──────────┬────────╮ │ │", "│ │ │ crates-io │ │ reedline │ {recor │ │ │", @@ -1274,37 +1300,11 @@ fn test_expand_big_0() { "│ │ │ │ │ │ elds} │ │ │", "│ │ │ │ ╰──────────┴────────╯ │ │", "│ │ ╰───────────┴───────────────────────╯ │", - "│ target │ {record 3 fields} │", - "│ │ ╭─────────┬─────────────────────────╮ │", - "│ workspace │ │ │ ╭────┬────────────────╮ │ │", - "│ │ │ members │ │ 0 │ crates/nu-cli │ │ │", - "│ │ │ │ │ 1 │ crates/nu-engi │ │ │", - "│ │ │ │ │ │ ne │ │ │", - "│ │ │ │ │ 2 │ crates/nu-pars │ │ │", - "│ │ │ │ │ │ er │ │ │", - "│ │ │ │ │ 3 │ crates/nu-syst │ │ │", - "│ │ │ │ │ │ em │ │ │", - "│ │ │ │ │ 4 │ crates/nu-comm │ │ │", - "│ │ │ │ │ │ and │ │ │", - "│ │ │ │ │ 5 │ crates/nu-prot │ │ │", - "│ │ │ │ │ │ ocol │ │ │", - "│ │ │ │ │ 6 │ crates/nu-plug │ │ │", - "│ │ │ │ │ │ in │ │ │", - "│ │ │ │ │ 7 │ crates/nu_plug │ │ │", - "│ │ │ │ │ │ in_inc │ │ │", - "│ │ │ │ │ 8 │ crates/nu_plug │ │ │", - "│ │ │ │ │ │ in_gstat │ │ │", - "│ │ │ │ │ 9 │ crates/nu_plug │ │ │", - "│ │ │ │ │ │ in_example │ │ │", - "│ │ │ │ │ 10 │ crates/nu_plug │ │ │", - "│ │ │ │ │ │ in_query │ │ │", - "│ │ │ │ │ 11 │ crates/nu_plug │ │ │", - "│ │ │ │ │ │ in_custom_valu │ │ │", - "│ │ │ │ │ │ es │ │ │", - "│ │ │ │ │ 12 │ crates/nu-util │ │ │", - "│ │ │ │ │ │ s │ │ │", - "│ │ │ │ ╰────┴────────────────╯ │ │", - "│ │ ╰─────────┴─────────────────────────╯ │", + "│ │ ╭───┬────────────┬─────────╮ │", + "│ bench │ │ # │ name │ harness │ │", + "│ │ ├───┼────────────┼─────────┤ │", + "│ │ │ 0 │ benchmarks │ false │ │", + "│ │ ╰───┴────────────┴─────────╯ │", "╰──────────────────┴───────────────────────────────────────╯", ]); @@ -1319,14 +1319,21 @@ fn test_expand_big_0() { let expected = join_lines([ "╭──────────────────┬───────────────────╮", - "│ bench │ [table 1 row] │", - "│ bin │ [table 1 row] │", - "│ dependencies │ {record 13 │", - "│ │ fields} │", - "│ dev-dependencies │ {record 9 fields} │", - "│ features │ {record 8 fields} │", "│ package │ {record 13 │", "│ │ fields} │", + "│ │ ╭─────────┬─────╮ │", + "│ workspace │ │ members │ [li │ │", + "│ │ │ │ st │ │", + "│ │ │ │ 13 │ │", + "│ │ │ │ ite │ │", + "│ │ │ │ ms] │ │", + "│ │ ╰─────────┴─────╯ │", + "│ dependencies │ {record 13 │", + "│ │ fields} │", + "│ target │ {record 3 fields} │", + "│ dev-dependencies │ {record 9 fields} │", + "│ features │ {record 8 fields} │", + "│ bin │ [table 1 row] │", "│ │ ╭───────────┬───╮ │", "│ patch │ │ crates-io │ { │ │", "│ │ │ │ r │ │", @@ -1345,14 +1352,7 @@ fn test_expand_big_0() { "│ │ │ │ d │ │", "│ │ │ │ } │ │", "│ │ ╰───────────┴───╯ │", - "│ target │ {record 3 fields} │", - "│ │ ╭─────────┬─────╮ │", - "│ workspace │ │ members │ [li │ │", - "│ │ │ │ st │ │", - "│ │ │ │ 13 │ │", - "│ │ │ │ ite │ │", - "│ │ │ │ ms] │ │", - "│ │ ╰─────────┴─────╯ │", + "│ bench │ [table 1 row] │", "╰──────────────────┴───────────────────╯", ]); @@ -1952,107 +1952,7 @@ fn test_collapse_big_0() { _print_lines(&actual.out, 80); let expected = join_lines([ - "╭──────────────────┬─────────┬─────────────────────────────────────────────────╮", - "│ bench │ harness │ name │", - "│ ├─────────┼─────────────────────────────────────────────────┤", - "│ │ false │ benchmarks │", - "├──────────────────┼──────┬──┴─────────────────────────────────────────────────┤", - "│ bin │ name │ path │", - "│ ├──────┼────────────────────────────────────────────────────┤", - "│ │ nu │ src/main.rs │", - "├──────────────────┼──────┴────────┬──────────┬────────────────────────────────┤", - "│ dependencies │ chrono │ features │ serde │", - "│ │ ├──────────┼────────────────────────────────┤", - "│ │ │ version │ 0.4.23 │", - "│ ├───────────────┼──────────┴────────────────────────────────┤", - "│ │ crossterm │ 0.24.0 │", - "│ ├───────────────┼───────────────────────────────────────────┤", - "│ │ ctrlc │ 3.2.1 │", - "│ ├───────────────┼───────────────────────────────────────────┤", - "│ │ is_executable │ 1.0.1 │", - "│ ├───────────────┼───────────────────────────────────────────┤", - "│ │ log │ 0.4 │", - "│ ├───────────────┼──────────┬────────────────────────────────┤", - "│ │ miette │ features │ fancy-no-backtrace │", - "│ │ ├──────────┼────────────────────────────────┤", - "│ │ │ version │ 5.5.0 │", - "│ ├───────────────┼──────────┴────────────────────────────────┤", - "│ │ nu-ansi-term │ 0.46.0 │", - "│ ├───────────────┼─────────┬─────────────────────────────────┤", - "│ │ nu-cli │ path │ ./crates/nu-cli │", - "│ │ ├─────────┼─────────────────────────────────┤", - "│ │ │ version │ 0.74.1 │", - "│ ├───────────────┼─────────┼─────────────────────────────────┤", - "│ │ nu-engine │ path │ ./crates/nu-engine │", - "│ │ ├─────────┼─────────────────────────────────┤", - "│ │ │ version │ 0.74.1 │", - "│ ├───────────────┼─────────┴─────────────────────────────────┤", - "│ │ rayon │ 1.6.1 │", - "│ ├───────────────┼──────────┬────────────────────────────────┤", - "│ │ reedline │ features │ bashisms │", - "│ │ │ ├────────────────────────────────┤", - "│ │ │ │ sqlite │", - "│ │ ├──────────┼────────────────────────────────┤", - "│ │ │ version │ 0.14.0 │", - "│ ├───────────────┼──────────┴────────────────────────────────┤", - "│ │ simplelog │ 0.12.0 │", - "│ ├───────────────┼───────────────────────────────────────────┤", - "│ │ time │ 0.3.12 │", - "├──────────────────┼───────────────┴───┬───────────────────────────────────────┤", - "│ dev-dependencies │ assert_cmd │ 2.0.2 │", - "│ ├───────────────────┼───────────────────────────────────────┤", - "│ │ criterion │ 0.4 │", - "│ ├───────────────────┼───────────────────────────────────────┤", - "│ │ hamcrest2 │ 0.3.0 │", - "│ ├───────────────────┼───────────────────────────────────────┤", - "│ │ itertools │ 0.10.3 │", - "│ ├───────────────────┼─────────┬─────────────────────────────┤", - "│ │ nu-test-support │ path │ ./crates/nu-test-support │", - "│ │ ├─────────┼─────────────────────────────┤", - "│ │ │ version │ 0.74.1 │", - "│ ├───────────────────┼─────────┴─────────────────────────────┤", - "│ │ pretty_assertions │ 1.0.0 │", - "│ ├───────────────────┼──────────────────┬────────────────────┤", - "│ │ rstest │ default-features │ false │", - "│ │ ├──────────────────┼────────────────────┤", - "│ │ │ version │ 0.15.0 │", - "│ ├───────────────────┼──────────────────┴────────────────────┤", - "│ │ serial_test │ 0.10.0 │", - "│ ├───────────────────┼───────────────────────────────────────┤", - "│ │ tempfile │ 3.2.0 │", - "├──────────────────┼───────────────────┴─┬─────────────────────────────────────┤", - "│ features │ default │ plugin │", - "│ │ ├─────────────────────────────────────┤", - "│ │ │ which-support │", - "│ │ ├─────────────────────────────────────┤", - "│ │ │ trash-support │", - "│ │ ├─────────────────────────────────────┤", - "│ │ │ sqlite │", - "│ ├─────────────────────┼─────────────────────────────────────┤", - "│ │ extra │ default │", - "│ ├─────────────────────┼─────────────────────────────────────┤", - "│ │ plugin │ nu-plugin │", - "│ │ ├─────────────────────────────────────┤", - "│ │ │ nu-cli/plugin │", - "│ │ ├─────────────────────────────────────┤", - "│ │ │ nu-parser/plugin │", - "│ │ ├─────────────────────────────────────┤", - "│ │ │ nu-command/plugin │", - "│ │ ├─────────────────────────────────────┤", - "│ │ │ nu-protocol/plugin │", - "│ │ ├─────────────────────────────────────┤", - "│ │ │ nu-engine/plugin │", - "│ ├─────────────────────┼─────────────────────────────────────┤", - "│ │ stable │ default │", - "│ ├─────────────────────┼─────────────────────────────────────┤", - "│ │ static-link-openssl │ dep:openssl │", - "│ ├─────────────────────┼─────────────────────────────────────┤", - "│ │ trash-support │ nu-command/trash-support │", - "│ ├─────────────────────┼─────────────────────────────────────┤", - "│ │ wasi │ │", - "│ ├─────────────────────┼─────────────────────────────────────┤", - "│ │ which-support │ nu-command/which-support │", - "├──────────────────┼───────────────┬─────┴─────────────────────────────────────┤", + "╭──────────────────┬───────────────┬───────────────────────────────────────────╮", "│ package │ authors │ The Nushell Project Developers │", "│ ├───────────────┼───────────────────────────────────────────┤", "│ │ default-run │ nu │", @@ -2068,19 +1968,7 @@ fn test_collapse_big_0() { "│ │ homepage │ https://www.nushell.sh │", "│ ├───────────────┼───────────────────────────────────────────┤", "│ │ license │ MIT │", - "│ ├───────────────┼──────────┬───────────┬────────────────────┤", - "│ │ metadata │ binstall │ overrides │ ... │", - "│ │ │ ├───────────┼────────────────────┤", - "│ │ │ │ pkg-fmt │ tgz │", - "│ │ │ ├───────────┼────────────────────┤", - "│ │ │ │ pkg-url │ { repo }/releases/ │", - "│ │ │ │ │ download/{ v │", - "│ │ │ │ │ ersion │", - "│ │ │ │ │ }/{ name }-{ vers │", - "│ │ │ │ │ ion }- │", - "│ │ │ │ │ { target }.{ │", - "│ │ │ │ │ archive-format } │", - "│ ├───────────────┼──────────┴───────────┴────────────────────┤", + "│ ├───────────────┼───────────────────────────────────────────┤", "│ │ name │ nu │", "│ ├───────────────┼───────────────────────────────────────────┤", "│ │ repository │ https://github.com/nushell/nushell │", @@ -2088,22 +1976,19 @@ fn test_collapse_big_0() { "│ │ rust-version │ 1.60 │", "│ ├───────────────┼───────────────────────────────────────────┤", "│ │ version │ 0.74.1 │", - "├──────────────────┼───────────┬───┴──────┬────────┬───────────────────────────┤", - "│ patch │ crates-io │ reedline │ branch │ main │", - "│ │ │ ├────────┼───────────────────────────┤", - "│ │ │ │ git │ https://github.com/nushel │", - "│ │ │ │ │ l/reedline.git │", - "├──────────────────┼───────────┴──────────┴────────┴─┬──────────────┬──────────┤", - "│ target │ cfg(not(target_os = \"windows\")) │ dependencies │ ... │", - "│ │ │ ├──────────┤", - "│ │ │ │ ... │", - "│ ├─────────────────────────────────┼──────────────┼──────────┤", - "│ │ cfg(target_family = \"unix\") │ dependencies │ ... │", - "│ │ │ ├──────────┤", - "│ │ │ │ ... │", - "│ ├─────────────────────────────────┼──────────────┴──────────┤", - "│ │ cfg(windows) │ ... │", - "├──────────────────┼─────────┬───────────────────────┴─────────────────────────┤", + "│ ├───────────────┼──────────┬───────────┬────────────────────┤", + "│ │ metadata │ binstall │ pkg-url │ { repo }/releases/ │", + "│ │ │ │ │ download/{ v │", + "│ │ │ │ │ ersion │", + "│ │ │ │ │ }/{ name }-{ vers │", + "│ │ │ │ │ ion }- │", + "│ │ │ │ │ { target }.{ │", + "│ │ │ │ │ archive-format } │", + "│ │ │ ├───────────┼────────────────────┤", + "│ │ │ │ pkg-fmt │ tgz │", + "│ │ │ ├───────────┼────────────────────┤", + "│ │ │ │ overrides │ ... │", + "├──────────────────┼─────────┬─────┴──────────┴───────────┴────────────────────┤", "│ workspace │ members │ crates/nu-cli │", "│ │ ├─────────────────────────────────────────────────┤", "│ │ │ crates/nu-engine │", @@ -2129,7 +2014,122 @@ fn test_collapse_big_0() { "│ │ │ crates/nu_plugin_custom_values │", "│ │ ├─────────────────────────────────────────────────┤", "│ │ │ crates/nu-utils │", - "╰──────────────────┴─────────┴─────────────────────────────────────────────────╯", + "├──────────────────┼─────────┴─────┬──────────┬────────────────────────────────┤", + "│ dependencies │ chrono │ version │ 0.4.23 │", + "│ │ ├──────────┼────────────────────────────────┤", + "│ │ │ features │ serde │", + "│ ├───────────────┼──────────┴────────────────────────────────┤", + "│ │ crossterm │ 0.24.0 │", + "│ ├───────────────┼───────────────────────────────────────────┤", + "│ │ ctrlc │ 3.2.1 │", + "│ ├───────────────┼───────────────────────────────────────────┤", + "│ │ log │ 0.4 │", + "│ ├───────────────┼──────────┬────────────────────────────────┤", + "│ │ miette │ version │ 5.5.0 │", + "│ │ ├──────────┼────────────────────────────────┤", + "│ │ │ features │ fancy-no-backtrace │", + "│ ├───────────────┼──────────┴────────────────────────────────┤", + "│ │ nu-ansi-term │ 0.46.0 │", + "│ ├───────────────┼─────────┬─────────────────────────────────┤", + "│ │ nu-cli │ path │ ./crates/nu-cli │", + "│ │ ├─────────┼─────────────────────────────────┤", + "│ │ │ version │ 0.74.1 │", + "│ ├───────────────┼─────────┼─────────────────────────────────┤", + "│ │ nu-engine │ path │ ./crates/nu-engine │", + "│ │ ├─────────┼─────────────────────────────────┤", + "│ │ │ version │ 0.74.1 │", + "│ ├───────────────┼─────────┴┬────────────────────────────────┤", + "│ │ reedline │ version │ 0.14.0 │", + "│ │ ├──────────┼────────────────────────────────┤", + "│ │ │ features │ bashisms │", + "│ │ │ ├────────────────────────────────┤", + "│ │ │ │ sqlite │", + "│ ├───────────────┼──────────┴────────────────────────────────┤", + "│ │ rayon │ 1.6.1 │", + "│ ├───────────────┼───────────────────────────────────────────┤", + "│ │ is_executable │ 1.0.1 │", + "│ ├───────────────┼───────────────────────────────────────────┤", + "│ │ simplelog │ 0.12.0 │", + "│ ├───────────────┼───────────────────────────────────────────┤", + "│ │ time │ 0.3.12 │", + "├──────────────────┼───────────────┴─────────────────┬──────────────┬──────────┤", + "│ target │ cfg(not(target_os = \"windows\")) │ dependencies │ ... │", + "│ │ │ ├──────────┤", + "│ │ │ │ ... │", + "│ ├─────────────────────────────────┼──────────────┴──────────┤", + "│ │ cfg(windows) │ ... │", + "│ ├─────────────────────────────────┼──────────────┬──────────┤", + "│ │ cfg(target_family = \"unix\") │ dependencies │ ... │", + "│ │ │ ├──────────┤", + "│ │ │ │ ... │", + "├──────────────────┼───────────────────┬─────────┬───┴──────────────┴──────────┤", + "│ dev-dependencies │ nu-test-support │ path │ ./crates/nu-test-support │", + "│ │ ├─────────┼─────────────────────────────┤", + "│ │ │ version │ 0.74.1 │", + "│ ├───────────────────┼─────────┴─────────────────────────────┤", + "│ │ tempfile │ 3.2.0 │", + "│ ├───────────────────┼───────────────────────────────────────┤", + "│ │ assert_cmd │ 2.0.2 │", + "│ ├───────────────────┼───────────────────────────────────────┤", + "│ │ criterion │ 0.4 │", + "│ ├───────────────────┼───────────────────────────────────────┤", + "│ │ pretty_assertions │ 1.0.0 │", + "│ ├───────────────────┼───────────────────────────────────────┤", + "│ │ serial_test │ 0.10.0 │", + "│ ├───────────────────┼───────────────────────────────────────┤", + "│ │ hamcrest2 │ 0.3.0 │", + "│ ├───────────────────┼──────────────────┬────────────────────┤", + "│ │ rstest │ version │ 0.15.0 │", + "│ │ ├──────────────────┼────────────────────┤", + "│ │ │ default-features │ false │", + "│ ├───────────────────┼──────────────────┴────────────────────┤", + "│ │ itertools │ 0.10.3 │", + "├──────────────────┼───────────────────┴─┬─────────────────────────────────────┤", + "│ features │ plugin │ nu-plugin │", + "│ │ ├─────────────────────────────────────┤", + "│ │ │ nu-cli/plugin │", + "│ │ ├─────────────────────────────────────┤", + "│ │ │ nu-parser/plugin │", + "│ │ ├─────────────────────────────────────┤", + "│ │ │ nu-command/plugin │", + "│ │ ├─────────────────────────────────────┤", + "│ │ │ nu-protocol/plugin │", + "│ │ ├─────────────────────────────────────┤", + "│ │ │ nu-engine/plugin │", + "│ ├─────────────────────┼─────────────────────────────────────┤", + "│ │ extra │ default │", + "│ ├─────────────────────┼─────────────────────────────────────┤", + "│ │ default │ plugin │", + "│ │ ├─────────────────────────────────────┤", + "│ │ │ which-support │", + "│ │ ├─────────────────────────────────────┤", + "│ │ │ trash-support │", + "│ │ ├─────────────────────────────────────┤", + "│ │ │ sqlite │", + "│ ├─────────────────────┼─────────────────────────────────────┤", + "│ │ stable │ default │", + "│ ├─────────────────────┼─────────────────────────────────────┤", + "│ │ wasi │ │", + "│ ├─────────────────────┼─────────────────────────────────────┤", + "│ │ static-link-openssl │ dep:openssl │", + "│ ├─────────────────────┼─────────────────────────────────────┤", + "│ │ which-support │ nu-command/which-support │", + "│ ├─────────────────────┼─────────────────────────────────────┤", + "│ │ trash-support │ nu-command/trash-support │", + "├──────────────────┼──────┬──────────────┴─────────────────────────────────────┤", + "│ bin │ name │ path │", + "│ ├──────┼────────────────────────────────────────────────────┤", + "│ │ nu │ src/main.rs │", + "├──────────────────┼──────┴────┬──────────┬────────┬───────────────────────────┤", + "│ patch │ crates-io │ reedline │ git │ https://github.com/nushel │", + "│ │ │ │ │ l/reedline.git │", + "│ │ │ ├────────┼───────────────────────────┤", + "│ │ │ │ branch │ main │", + "├──────────────────┼───────────┴┬─────────┴────────┴───────────────────────────┤", + "│ bench │ name │ harness │", + "│ ├────────────┼──────────────────────────────────────────────┤", + "│ │ benchmarks │ false │", + "╰──────────────────┴────────────┴──────────────────────────────────────────────╯", ]); assert_eq!(actual.out, expected); @@ -2142,107 +2142,7 @@ fn test_collapse_big_0() { _print_lines(&actual.out, 111); let expected = join_lines([ - "╭──────────────────┬─────────┬────────────────────────────────────────────────────────────────────────────────╮", - "│ bench │ harness │ name │", - "│ ├─────────┼────────────────────────────────────────────────────────────────────────────────┤", - "│ │ false │ benchmarks │", - "├──────────────────┼──────┬──┴────────────────────────────────────────────────────────────────────────────────┤", - "│ bin │ name │ path │", - "│ ├──────┼───────────────────────────────────────────────────────────────────────────────────┤", - "│ │ nu │ src/main.rs │", - "├──────────────────┼──────┴────────┬──────────┬───────────────────────────────────────────────────────────────┤", - "│ dependencies │ chrono │ features │ serde │", - "│ │ ├──────────┼───────────────────────────────────────────────────────────────┤", - "│ │ │ version │ 0.4.23 │", - "│ ├───────────────┼──────────┴───────────────────────────────────────────────────────────────┤", - "│ │ crossterm │ 0.24.0 │", - "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", - "│ │ ctrlc │ 3.2.1 │", - "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", - "│ │ is_executable │ 1.0.1 │", - "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", - "│ │ log │ 0.4 │", - "│ ├───────────────┼──────────┬───────────────────────────────────────────────────────────────┤", - "│ │ miette │ features │ fancy-no-backtrace │", - "│ │ ├──────────┼───────────────────────────────────────────────────────────────┤", - "│ │ │ version │ 5.5.0 │", - "│ ├───────────────┼──────────┴───────────────────────────────────────────────────────────────┤", - "│ │ nu-ansi-term │ 0.46.0 │", - "│ ├───────────────┼─────────┬────────────────────────────────────────────────────────────────┤", - "│ │ nu-cli │ path │ ./crates/nu-cli │", - "│ │ ├─────────┼────────────────────────────────────────────────────────────────┤", - "│ │ │ version │ 0.74.1 │", - "│ ├───────────────┼─────────┼────────────────────────────────────────────────────────────────┤", - "│ │ nu-engine │ path │ ./crates/nu-engine │", - "│ │ ├─────────┼────────────────────────────────────────────────────────────────┤", - "│ │ │ version │ 0.74.1 │", - "│ ├───────────────┼─────────┴────────────────────────────────────────────────────────────────┤", - "│ │ rayon │ 1.6.1 │", - "│ ├───────────────┼──────────┬───────────────────────────────────────────────────────────────┤", - "│ │ reedline │ features │ bashisms │", - "│ │ │ ├───────────────────────────────────────────────────────────────┤", - "│ │ │ │ sqlite │", - "│ │ ├──────────┼───────────────────────────────────────────────────────────────┤", - "│ │ │ version │ 0.14.0 │", - "│ ├───────────────┼──────────┴───────────────────────────────────────────────────────────────┤", - "│ │ simplelog │ 0.12.0 │", - "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", - "│ │ time │ 0.3.12 │", - "├──────────────────┼───────────────┴───┬──────────────────────────────────────────────────────────────────────┤", - "│ dev-dependencies │ assert_cmd │ 2.0.2 │", - "│ ├───────────────────┼──────────────────────────────────────────────────────────────────────┤", - "│ │ criterion │ 0.4 │", - "│ ├───────────────────┼──────────────────────────────────────────────────────────────────────┤", - "│ │ hamcrest2 │ 0.3.0 │", - "│ ├───────────────────┼──────────────────────────────────────────────────────────────────────┤", - "│ │ itertools │ 0.10.3 │", - "│ ├───────────────────┼─────────┬────────────────────────────────────────────────────────────┤", - "│ │ nu-test-support │ path │ ./crates/nu-test-support │", - "│ │ ├─────────┼────────────────────────────────────────────────────────────┤", - "│ │ │ version │ 0.74.1 │", - "│ ├───────────────────┼─────────┴────────────────────────────────────────────────────────────┤", - "│ │ pretty_assertions │ 1.0.0 │", - "│ ├───────────────────┼──────────────────┬───────────────────────────────────────────────────┤", - "│ │ rstest │ default-features │ false │", - "│ │ ├──────────────────┼───────────────────────────────────────────────────┤", - "│ │ │ version │ 0.15.0 │", - "│ ├───────────────────┼──────────────────┴───────────────────────────────────────────────────┤", - "│ │ serial_test │ 0.10.0 │", - "│ ├───────────────────┼──────────────────────────────────────────────────────────────────────┤", - "│ │ tempfile │ 3.2.0 │", - "├──────────────────┼───────────────────┴─┬────────────────────────────────────────────────────────────────────┤", - "│ features │ default │ plugin │", - "│ │ ├────────────────────────────────────────────────────────────────────┤", - "│ │ │ which-support │", - "│ │ ├────────────────────────────────────────────────────────────────────┤", - "│ │ │ trash-support │", - "│ │ ├────────────────────────────────────────────────────────────────────┤", - "│ │ │ sqlite │", - "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", - "│ │ extra │ default │", - "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", - "│ │ plugin │ nu-plugin │", - "│ │ ├────────────────────────────────────────────────────────────────────┤", - "│ │ │ nu-cli/plugin │", - "│ │ ├────────────────────────────────────────────────────────────────────┤", - "│ │ │ nu-parser/plugin │", - "│ │ ├────────────────────────────────────────────────────────────────────┤", - "│ │ │ nu-command/plugin │", - "│ │ ├────────────────────────────────────────────────────────────────────┤", - "│ │ │ nu-protocol/plugin │", - "│ │ ├────────────────────────────────────────────────────────────────────┤", - "│ │ │ nu-engine/plugin │", - "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", - "│ │ stable │ default │", - "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", - "│ │ static-link-openssl │ dep:openssl │", - "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", - "│ │ trash-support │ nu-command/trash-support │", - "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", - "│ │ wasi │ │", - "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", - "│ │ which-support │ nu-command/which-support │", - "├──────────────────┼───────────────┬─────┴────────────────────────────────────────────────────────────────────┤", + "╭──────────────────┬───────────────┬──────────────────────────────────────────────────────────────────────────╮", "│ package │ authors │ The Nushell Project Developers │", "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", "│ │ default-run │ nu │", @@ -2258,15 +2158,7 @@ fn test_collapse_big_0() { "│ │ homepage │ https://www.nushell.sh │", "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", "│ │ license │ MIT │", - "│ ├───────────────┼──────────┬───────────┬────────────────────────┬─────────┬────────────────┤", - "│ │ metadata │ binstall │ overrides │ x86_64-pc-windows-msvc │ pkg-fmt │ zip │", - "│ │ │ ├───────────┼────────────────────────┴─────────┴────────────────┤", - "│ │ │ │ pkg-fmt │ tgz │", - "│ │ │ ├───────────┼───────────────────────────────────────────────────┤", - "│ │ │ │ pkg-url │ { repo }/releases/download/{ v │", - "│ │ │ │ │ ersion }/{ name }-{ version }- │", - "│ │ │ │ │ { target }.{ archive-format } │", - "│ ├───────────────┼──────────┴───────────┴───────────────────────────────────────────────────┤", + "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", "│ │ name │ nu │", "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", "│ │ repository │ https://github.com/nushell/nushell │", @@ -2274,37 +2166,15 @@ fn test_collapse_big_0() { "│ │ rust-version │ 1.60 │", "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", "│ │ version │ 0.74.1 │", - "├──────────────────┼───────────┬───┴──────┬────────┬──────────────────────────────────────────────────────────┤", - "│ patch │ crates-io │ reedline │ branch │ main │", - "│ │ │ ├────────┼──────────────────────────────────────────────────────────┤", - "│ │ │ │ git │ https://github.com/nushell/reedline.git │", - "├──────────────────┼───────────┴──────────┴────────┴─┬──────────────┬─────────────┬──────────┬────────────────┤", - "│ target │ cfg(not(target_os = \"windows\")) │ dependencies │ openssl │ features │ vendored │", - "│ │ │ │ ├──────────┼────────────────┤", - "│ │ │ │ │ optional │ true │", - "│ │ │ │ ├──────────┼────────────────┤", - "│ │ │ │ │ version │ 0.10.38 │", - "│ │ │ ├─────────────┼──────────┴───────┬────────┤", - "│ │ │ │ signal-hook │ default-features │ false │", - "│ │ │ │ ├──────────────────┼────────┤", - "│ │ │ │ │ version │ 0.3.14 │", - "│ ├─────────────────────────────────┼──────────────┼──────┬──────┴──────────────────┴────────┤", - "│ │ cfg(target_family = \"unix\") │ dependencies │ atty │ 0.2 │", - "│ │ │ ├──────┼──────────────────┬───────────────┤", - "│ │ │ │ nix │ default-features │ false │", - "│ │ │ │ ├──────────────────┼───────────────┤", - "│ │ │ │ │ features │ signal │", - "│ │ │ │ │ ├───────────────┤", - "│ │ │ │ │ │ process │", - "│ │ │ │ │ ├───────────────┤", - "│ │ │ │ │ │ fs │", - "│ │ │ │ │ ├───────────────┤", - "│ │ │ │ │ │ term │", - "│ │ │ │ ├──────────────────┼───────────────┤", - "│ │ │ │ │ version │ 0.25 │", - "│ ├─────────────────────────────────┼──────────────┴─────┬┴───────┬──────────┴───────────────┤", - "│ │ cfg(windows) │ build-dependencies │ winres │ 0.1 │", - "├──────────────────┼─────────┬───────────────────────┴────────────────────┴────────┴──────────────────────────┤", + "│ ├───────────────┼──────────┬───────────┬───────────────────────────────────────────────────┤", + "│ │ metadata │ binstall │ pkg-url │ { repo }/releases/download/{ v │", + "│ │ │ │ │ ersion }/{ name }-{ version }- │", + "│ │ │ │ │ { target }.{ archive-format } │", + "│ │ │ ├───────────┼───────────────────────────────────────────────────┤", + "│ │ │ │ pkg-fmt │ tgz │", + "│ │ │ ├───────────┼────────────────────────┬─────────┬────────────────┤", + "│ │ │ │ overrides │ x86_64-pc-windows-msvc │ pkg-fmt │ zip │", + "├──────────────────┼─────────┬─────┴──────────┴───────────┴────────────────────────┴─────────┴────────────────┤", "│ workspace │ members │ crates/nu-cli │", "│ │ ├────────────────────────────────────────────────────────────────────────────────┤", "│ │ │ crates/nu-engine │", @@ -2330,7 +2200,137 @@ fn test_collapse_big_0() { "│ │ │ crates/nu_plugin_custom_values │", "│ │ ├────────────────────────────────────────────────────────────────────────────────┤", "│ │ │ crates/nu-utils │", - "╰──────────────────┴─────────┴────────────────────────────────────────────────────────────────────────────────╯", + "├──────────────────┼─────────┴─────┬──────────┬───────────────────────────────────────────────────────────────┤", + "│ dependencies │ chrono │ version │ 0.4.23 │", + "│ │ ├──────────┼───────────────────────────────────────────────────────────────┤", + "│ │ │ features │ serde │", + "│ ├───────────────┼──────────┴───────────────────────────────────────────────────────────────┤", + "│ │ crossterm │ 0.24.0 │", + "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", + "│ │ ctrlc │ 3.2.1 │", + "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", + "│ │ log │ 0.4 │", + "│ ├───────────────┼──────────┬───────────────────────────────────────────────────────────────┤", + "│ │ miette │ version │ 5.5.0 │", + "│ │ ├──────────┼───────────────────────────────────────────────────────────────┤", + "│ │ │ features │ fancy-no-backtrace │", + "│ ├───────────────┼──────────┴───────────────────────────────────────────────────────────────┤", + "│ │ nu-ansi-term │ 0.46.0 │", + "│ ├───────────────┼─────────┬────────────────────────────────────────────────────────────────┤", + "│ │ nu-cli │ path │ ./crates/nu-cli │", + "│ │ ├─────────┼────────────────────────────────────────────────────────────────┤", + "│ │ │ version │ 0.74.1 │", + "│ ├───────────────┼─────────┼────────────────────────────────────────────────────────────────┤", + "│ │ nu-engine │ path │ ./crates/nu-engine │", + "│ │ ├─────────┼────────────────────────────────────────────────────────────────┤", + "│ │ │ version │ 0.74.1 │", + "│ ├───────────────┼─────────┴┬───────────────────────────────────────────────────────────────┤", + "│ │ reedline │ version │ 0.14.0 │", + "│ │ ├──────────┼───────────────────────────────────────────────────────────────┤", + "│ │ │ features │ bashisms │", + "│ │ │ ├───────────────────────────────────────────────────────────────┤", + "│ │ │ │ sqlite │", + "│ ├───────────────┼──────────┴───────────────────────────────────────────────────────────────┤", + "│ │ rayon │ 1.6.1 │", + "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", + "│ │ is_executable │ 1.0.1 │", + "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", + "│ │ simplelog │ 0.12.0 │", + "│ ├───────────────┼──────────────────────────────────────────────────────────────────────────┤", + "│ │ time │ 0.3.12 │", + "├──────────────────┼───────────────┴─────────────────┬──────────────┬─────────────┬──────────┬────────────────┤", + "│ target │ cfg(not(target_os = \"windows\")) │ dependencies │ openssl │ version │ 0.10.38 │", + "│ │ │ │ ├──────────┼────────────────┤", + "│ │ │ │ │ features │ vendored │", + "│ │ │ │ ├──────────┼────────────────┤", + "│ │ │ │ │ optional │ true │", + "│ │ │ ├─────────────┼──────────┴───────┬────────┤", + "│ │ │ │ signal-hook │ version │ 0.3.14 │", + "│ │ │ │ ├──────────────────┼────────┤", + "│ │ │ │ │ default-features │ false │", + "│ ├─────────────────────────────────┼──────────────┴─────┬───────┴┬─────────────────┴────────┤", + "│ │ cfg(windows) │ build-dependencies │ winres │ 0.1 │", + "│ ├─────────────────────────────────┼──────────────┬─────┴┬───────┴──────────┬───────────────┤", + "│ │ cfg(target_family = \"unix\") │ dependencies │ nix │ version │ 0.25 │", + "│ │ │ │ ├──────────────────┼───────────────┤", + "│ │ │ │ │ default-features │ false │", + "│ │ │ │ ├──────────────────┼───────────────┤", + "│ │ │ │ │ features │ signal │", + "│ │ │ │ │ ├───────────────┤", + "│ │ │ │ │ │ process │", + "│ │ │ │ │ ├───────────────┤", + "│ │ │ │ │ │ fs │", + "│ │ │ │ │ ├───────────────┤", + "│ │ │ │ │ │ term │", + "│ │ │ ├──────┼──────────────────┴───────────────┤", + "│ │ │ │ atty │ 0.2 │", + "├──────────────────┼───────────────────┬─────────┬───┴──────────────┴──────┴──────────────────────────────────┤", + "│ dev-dependencies │ nu-test-support │ path │ ./crates/nu-test-support │", + "│ │ ├─────────┼────────────────────────────────────────────────────────────┤", + "│ │ │ version │ 0.74.1 │", + "│ ├───────────────────┼─────────┴────────────────────────────────────────────────────────────┤", + "│ │ tempfile │ 3.2.0 │", + "│ ├───────────────────┼──────────────────────────────────────────────────────────────────────┤", + "│ │ assert_cmd │ 2.0.2 │", + "│ ├───────────────────┼──────────────────────────────────────────────────────────────────────┤", + "│ │ criterion │ 0.4 │", + "│ ├───────────────────┼──────────────────────────────────────────────────────────────────────┤", + "│ │ pretty_assertions │ 1.0.0 │", + "│ ├───────────────────┼──────────────────────────────────────────────────────────────────────┤", + "│ │ serial_test │ 0.10.0 │", + "│ ├───────────────────┼──────────────────────────────────────────────────────────────────────┤", + "│ │ hamcrest2 │ 0.3.0 │", + "│ ├───────────────────┼──────────────────┬───────────────────────────────────────────────────┤", + "│ │ rstest │ version │ 0.15.0 │", + "│ │ ├──────────────────┼───────────────────────────────────────────────────┤", + "│ │ │ default-features │ false │", + "│ ├───────────────────┼──────────────────┴───────────────────────────────────────────────────┤", + "│ │ itertools │ 0.10.3 │", + "├──────────────────┼───────────────────┴─┬────────────────────────────────────────────────────────────────────┤", + "│ features │ plugin │ nu-plugin │", + "│ │ ├────────────────────────────────────────────────────────────────────┤", + "│ │ │ nu-cli/plugin │", + "│ │ ├────────────────────────────────────────────────────────────────────┤", + "│ │ │ nu-parser/plugin │", + "│ │ ├────────────────────────────────────────────────────────────────────┤", + "│ │ │ nu-command/plugin │", + "│ │ ├────────────────────────────────────────────────────────────────────┤", + "│ │ │ nu-protocol/plugin │", + "│ │ ├────────────────────────────────────────────────────────────────────┤", + "│ │ │ nu-engine/plugin │", + "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", + "│ │ extra │ default │", + "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", + "│ │ default │ plugin │", + "│ │ ├────────────────────────────────────────────────────────────────────┤", + "│ │ │ which-support │", + "│ │ ├────────────────────────────────────────────────────────────────────┤", + "│ │ │ trash-support │", + "│ │ ├────────────────────────────────────────────────────────────────────┤", + "│ │ │ sqlite │", + "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", + "│ │ stable │ default │", + "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", + "│ │ wasi │ │", + "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", + "│ │ static-link-openssl │ dep:openssl │", + "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", + "│ │ which-support │ nu-command/which-support │", + "│ ├─────────────────────┼────────────────────────────────────────────────────────────────────┤", + "│ │ trash-support │ nu-command/trash-support │", + "├──────────────────┼──────┬──────────────┴────────────────────────────────────────────────────────────────────┤", + "│ bin │ name │ path │", + "│ ├──────┼───────────────────────────────────────────────────────────────────────────────────┤", + "│ │ nu │ src/main.rs │", + "├──────────────────┼──────┴────┬──────────┬────────┬──────────────────────────────────────────────────────────┤", + "│ patch │ crates-io │ reedline │ git │ https://github.com/nushell/reedline.git │", + "│ │ │ ├────────┼──────────────────────────────────────────────────────────┤", + "│ │ │ │ branch │ main │", + "├──────────────────┼───────────┴┬─────────┴────────┴──────────────────────────────────────────────────────────┤", + "│ bench │ name │ harness │", + "│ ├────────────┼─────────────────────────────────────────────────────────────────────────────┤", + "│ │ benchmarks │ false │", + "╰──────────────────┴────────────┴─────────────────────────────────────────────────────────────────────────────╯", ]); assert_eq!(actual.out, expected); @@ -2889,3 +2889,9 @@ fn table_list() { let actual = nu!("table --list --theme basic"); assert_eq!(actual.out, "╭────┬────────────────╮│ 0 │ basic ││ 1 │ compact ││ 2 │ compact_double ││ 3 │ default ││ 4 │ heavy ││ 5 │ light ││ 6 │ none ││ 7 │ reinforced ││ 8 │ rounded ││ 9 │ thin ││ 10 │ with_love ││ 11 │ psql ││ 12 │ markdown ││ 13 │ dots ││ 14 │ restructured ││ 15 │ ascii_rounded ││ 16 │ basic_compact │╰────┴────────────────╯"); } + +#[test] +fn table_kv_header_on_separator_trim_algorithm() { + let actual = nu!("$env.config.table.header_on_separator = true; {key1: '111111111111111111111111111111111111111111111111111111111111'} | table --width=60 --theme basic"); + assert_eq!(actual.out, "+------+---------------------------------------------------+| key1 | 1111111111111111111111111111111111111111111111111 || | 11111111111 |+------+---------------------------------------------------+"); +} diff --git a/crates/nu-command/tests/commands/take/rows.rs b/crates/nu-command/tests/commands/take/rows.rs index d5f3d1c601..6c34b61310 100644 --- a/crates/nu-command/tests/commands/take/rows.rs +++ b/crates/nu-command/tests/commands/take/rows.rs @@ -35,6 +35,20 @@ fn fails_on_string() { assert!(actual.err.contains("command doesn't support")); } +#[test] +fn takes_bytes() { + let actual = nu!("(0x[aa bb cc] | take 2) == 0x[aa bb]"); + + assert_eq!(actual.out, "true"); +} + +#[test] +fn takes_bytes_from_stream() { + let actual = nu!("(1.. | each { 0x[aa bb cc] } | bytes collect | take 2) == 0x[aa bb]"); + + assert_eq!(actual.out, "true"); +} + #[test] // covers a situation where `take` used to behave strangely on list input fn works_with_binary_list() { diff --git a/crates/nu-command/tests/commands/touch.rs b/crates/nu-command/tests/commands/touch.rs index 26d0325729..5f454cdec4 100644 --- a/crates/nu-command/tests/commands/touch.rs +++ b/crates/nu-command/tests/commands/touch.rs @@ -515,3 +515,16 @@ fn respects_cwd() { assert!(path.exists()); }) } + +#[test] +fn reference_respects_cwd() { + Playground::setup("touch_reference_respects_cwd", |dirs, _sandbox| { + nu!( + cwd: dirs.test(), + "mkdir 'dir'; cd 'dir'; touch 'ref.txt'; touch --reference 'ref.txt' 'foo.txt'" + ); + + let path = dirs.test().join("dir/foo.txt"); + assert!(path.exists()); + }) +} diff --git a/crates/nu-command/tests/commands/ucp.rs b/crates/nu-command/tests/commands/ucp.rs index 453786ad4e..728cf1dcb2 100644 --- a/crates/nu-command/tests/commands/ucp.rs +++ b/crates/nu-command/tests/commands/ucp.rs @@ -906,7 +906,7 @@ fn test_cp_debug_default() { #[cfg(any(target_os = "linux", target_os = "freebsd"))] if !actual .out - .contains("copy offload: unknown, reflink: unsupported, sparse detection: no") + .contains("copy offload: yes, reflink: unsupported, sparse detection: no") { panic!("{}", format!("Failure: stdout was \n{}", actual.out)); } diff --git a/crates/nu-command/tests/commands/use_.rs b/crates/nu-command/tests/commands/use_.rs index 8cb783f96b..63b1f8391a 100644 --- a/crates/nu-command/tests/commands/use_.rs +++ b/crates/nu-command/tests/commands/use_.rs @@ -189,9 +189,7 @@ fn use_module_creates_accurate_did_you_mean_1() { let actual = nu!(r#" module spam { export def foo [] { "foo" } }; use spam; foo "#); - assert!(actual.err.contains( - "command 'foo' was not found but it was imported from module 'spam'; try using `spam foo`" - )); + assert!(actual.err.contains("Did you mean `spam foo`")); } #[test] @@ -199,9 +197,9 @@ fn use_module_creates_accurate_did_you_mean_2() { let actual = nu!(r#" module spam { export def foo [] { "foo" } }; foo "#); - assert!(actual.err.contains( - "command 'foo' was not found but it exists in module 'spam'; try importing it with `use`" - )); + assert!(actual + .err + .contains("A command with that name exists in module `spam`")); } #[test] diff --git a/crates/nu-command/tests/format_conversions/csv.rs b/crates/nu-command/tests/format_conversions/csv.rs index a9be76d5c3..f10a84b672 100644 --- a/crates/nu-command/tests/format_conversions/csv.rs +++ b/crates/nu-command/tests/format_conversions/csv.rs @@ -284,7 +284,7 @@ fn from_csv_text_skipping_headers_to_table() { r#" open los_tres_amigos.txt | from csv --noheaders - | get column3 + | get column2 | length "# )); diff --git a/crates/nu-command/tests/format_conversions/json.rs b/crates/nu-command/tests/format_conversions/json.rs index 1989000df6..643e4c3f22 100644 --- a/crates/nu-command/tests/format_conversions/json.rs +++ b/crates/nu-command/tests/format_conversions/json.rs @@ -96,6 +96,32 @@ fn from_json_text_recognizing_objects_independently_to_table() { }) } +#[test] +fn from_json_text_objects_is_stream() { + Playground::setup("filter_from_json_test_2_is_stream", |dirs, sandbox| { + sandbox.with_files(&[FileWithContentToBeTrimmed( + "katz.txt", + r#" + {"name": "Yehuda", "rusty_luck": 1} + {"name": "JT", "rusty_luck": 1} + {"name": "Andres", "rusty_luck": 1} + {"name":"GorbyPuff", "rusty_luck": 3} + "#, + )]); + + let actual = nu!( + cwd: dirs.test(), pipeline( + r#" + open katz.txt + | from json -o + | describe -n + "# + )); + + assert_eq!(actual.out, "stream"); + }) +} + #[test] fn from_json_text_recognizing_objects_independently_to_table_strict() { Playground::setup("filter_from_json_test_2_strict", |dirs, sandbox| { diff --git a/crates/nu-command/tests/format_conversions/ssv.rs b/crates/nu-command/tests/format_conversions/ssv.rs index a673b122fb..5a62f796f7 100644 --- a/crates/nu-command/tests/format_conversions/ssv.rs +++ b/crates/nu-command/tests/format_conversions/ssv.rs @@ -74,7 +74,7 @@ fn from_ssv_text_treating_first_line_as_data_with_flag() { open oc_get_svc.txt | from ssv --noheaders -a | first - | get column1 + | get column0 "# )); @@ -84,7 +84,7 @@ fn from_ssv_text_treating_first_line_as_data_with_flag() { open oc_get_svc.txt | from ssv --noheaders | first - | get column1 + | get column0 "# )); diff --git a/crates/nu-command/tests/format_conversions/tsv.rs b/crates/nu-command/tests/format_conversions/tsv.rs index be57c60242..31740ef8d9 100644 --- a/crates/nu-command/tests/format_conversions/tsv.rs +++ b/crates/nu-command/tests/format_conversions/tsv.rs @@ -207,7 +207,7 @@ fn from_tsv_text_skipping_headers_to_table() { r#" open los_tres_amigos.txt | from tsv --noheaders - | get column3 + | get column2 | length "# )); diff --git a/crates/nu-command/tests/main.rs b/crates/nu-command/tests/main.rs index 72ccee3d85..1039e1a0d7 100644 --- a/crates/nu-command/tests/main.rs +++ b/crates/nu-command/tests/main.rs @@ -151,9 +151,8 @@ fn commands_declare_input_output_types() { let sig_name = cmd.signature().name; let category = cmd.signature().category; - if matches!(category, Category::Removed | Category::Custom(_)) { + if matches!(category, Category::Removed) { // Deprecated/Removed commands don't have to conform - // TODO: also upgrade the `--features dataframe` commands continue; } diff --git a/crates/nu-derive-value/Cargo.toml b/crates/nu-derive-value/Cargo.toml new file mode 100644 index 0000000000..50ca5d2407 --- /dev/null +++ b/crates/nu-derive-value/Cargo.toml @@ -0,0 +1,21 @@ +[package] +authors = ["The Nushell Project Developers"] +description = "Macros implementation of #[derive(FromValue, IntoValue)]" +edition = "2021" +license = "MIT" +name = "nu-derive-value" +repository = "https://github.com/nushell/nushell/tree/main/crates/nu-derive-value" +version = "0.95.1" + +[lib] +proc-macro = true +# we can only use exposed macros in doctests really, +# so we cannot test anything useful in a doctest +doctest = false + +[dependencies] +proc-macro2 = { workspace = true } +syn = { workspace = true } +quote = { workspace = true } +proc-macro-error = { workspace = true } +convert_case = { workspace = true } \ No newline at end of file diff --git a/crates/nu-cmd-dataframe/LICENSE b/crates/nu-derive-value/LICENSE similarity index 100% rename from crates/nu-cmd-dataframe/LICENSE rename to crates/nu-derive-value/LICENSE diff --git a/crates/nu-derive-value/src/attributes.rs b/crates/nu-derive-value/src/attributes.rs new file mode 100644 index 0000000000..6969b64287 --- /dev/null +++ b/crates/nu-derive-value/src/attributes.rs @@ -0,0 +1,116 @@ +use convert_case::Case; +use syn::{spanned::Spanned, Attribute, Fields, LitStr}; + +use crate::{error::DeriveError, HELPER_ATTRIBUTE}; + +#[derive(Debug)] +pub struct ContainerAttributes { + pub rename_all: Case, +} + +impl Default for ContainerAttributes { + fn default() -> Self { + Self { + rename_all: Case::Snake, + } + } +} + +impl ContainerAttributes { + pub fn parse_attrs<'a, M>( + iter: impl Iterator, + ) -> Result> { + let mut container_attrs = ContainerAttributes::default(); + for attr in filter(iter) { + // This is a container to allow returning derive errors inside the parse_nested_meta fn. + let mut err = Ok(()); + + attr.parse_nested_meta(|meta| { + let ident = meta.path.require_ident()?; + match ident.to_string().as_str() { + "rename_all" => { + // The matched case are all useful variants from `convert_case` with aliases + // that `serde` uses. + let case: LitStr = meta.value()?.parse()?; + let case = match case.value().as_str() { + "UPPER CASE" | "UPPER WITH SPACES CASE" => Case::Upper, + "lower case" | "lower with spaces case" => Case::Lower, + "Title Case" => Case::Title, + "camelCase" => Case::Camel, + "PascalCase" | "UpperCamelCase" => Case::Pascal, + "snake_case" => Case::Snake, + "UPPER_SNAKE_CASE" | "SCREAMING_SNAKE_CASE" => Case::UpperSnake, + "kebab-case" => Case::Kebab, + "COBOL-CASE" | "UPPER-KEBAB-CASE" | "SCREAMING-KEBAB-CASE" => { + Case::Cobol + } + "Train-Case" => Case::Train, + "flatcase" | "lowercase" => Case::Flat, + "UPPERFLATCASE" | "UPPERCASE" => Case::UpperFlat, + // Although very funny, we don't support `Case::{Toggle, Alternating}`, + // as we see no real benefit. + c => { + err = Err(DeriveError::InvalidAttributeValue { + value_span: case.span(), + value: Box::new(c.to_string()), + }); + return Ok(()); // We stored the err in `err`. + } + }; + container_attrs.rename_all = case; + } + ident => { + err = Err(DeriveError::UnexpectedAttribute { + meta_span: ident.span(), + }); + } + } + + Ok(()) + }) + .map_err(DeriveError::Syn)?; + + err?; // Shortcircuit here if `err` is holding some error. + } + + Ok(container_attrs) + } +} + +pub fn filter<'a>( + iter: impl Iterator, +) -> impl Iterator { + iter.filter(|attr| attr.path().is_ident(HELPER_ATTRIBUTE)) +} + +// The deny functions are built to easily deny the use of the helper attribute if used incorrectly. +// As the usage of it gets more complex, these functions might be discarded or replaced. + +/// Deny any attribute that uses the helper attribute. +pub fn deny(attrs: &[Attribute]) -> Result<(), DeriveError> { + match filter(attrs.iter()).next() { + Some(attr) => Err(DeriveError::InvalidAttributePosition { + attribute_span: attr.span(), + }), + None => Ok(()), + } +} + +/// Deny any attributes that uses the helper attribute on any field. +pub fn deny_fields(fields: &Fields) -> Result<(), DeriveError> { + match fields { + Fields::Named(fields) => { + for field in fields.named.iter() { + deny(&field.attrs)?; + } + } + Fields::Unnamed(fields) => { + for field in fields.unnamed.iter() { + deny(&field.attrs)?; + } + } + Fields::Unit => (), + } + + Ok(()) +} diff --git a/crates/nu-derive-value/src/error.rs b/crates/nu-derive-value/src/error.rs new file mode 100644 index 0000000000..b7fd3e2f91 --- /dev/null +++ b/crates/nu-derive-value/src/error.rs @@ -0,0 +1,82 @@ +use std::{any, fmt::Debug, marker::PhantomData}; + +use proc_macro2::Span; +use proc_macro_error::{Diagnostic, Level}; + +#[derive(Debug)] +pub enum DeriveError { + /// Marker variant, makes the `M` generic parameter valid. + _Marker(PhantomData), + + /// Parsing errors thrown by `syn`. + Syn(syn::parse::Error), + + /// `syn::DeriveInput` was a union, currently not supported + UnsupportedUnions, + + /// Only plain enums are supported right now. + UnsupportedEnums { fields_span: Span }, + + /// Found a `#[nu_value(x)]` attribute where `x` is unexpected. + UnexpectedAttribute { meta_span: Span }, + + /// Found a `#[nu_value(x)]` attribute at a invalid position. + InvalidAttributePosition { attribute_span: Span }, + + /// Found a valid `#[nu_value(x)]` attribute but the passed values is invalid. + InvalidAttributeValue { + value_span: Span, + value: Box, + }, +} + +impl From> for Diagnostic { + fn from(value: DeriveError) -> Self { + let derive_name = any::type_name::().split("::").last().expect("not empty"); + match value { + DeriveError::_Marker(_) => panic!("used marker variant"), + + DeriveError::Syn(e) => Diagnostic::spanned(e.span(), Level::Error, e.to_string()), + + DeriveError::UnsupportedUnions => Diagnostic::new( + Level::Error, + format!("`{derive_name}` cannot be derived from unions"), + ) + .help("consider refactoring to a struct".to_string()) + .note("if you really need a union, consider opening an issue on Github".to_string()), + + DeriveError::UnsupportedEnums { fields_span } => Diagnostic::spanned( + fields_span, + Level::Error, + format!("`{derive_name}` can only be derived from plain enums"), + ) + .help( + "consider refactoring your data type to a struct with a plain enum as a field" + .to_string(), + ) + .note("more complex enums could be implemented in the future".to_string()), + + DeriveError::InvalidAttributePosition { attribute_span } => Diagnostic::spanned( + attribute_span, + Level::Error, + "invalid attribute position".to_string(), + ) + .help(format!( + "check documentation for `{derive_name}` for valid placements" + )), + + DeriveError::UnexpectedAttribute { meta_span } => { + Diagnostic::spanned(meta_span, Level::Error, "unknown attribute".to_string()).help( + format!("check documentation for `{derive_name}` for valid attributes"), + ) + } + + DeriveError::InvalidAttributeValue { value_span, value } => { + Diagnostic::spanned(value_span, Level::Error, format!("invalid value {value:?}")) + .help(format!( + "check documentation for `{derive_name}` for valid attribute values" + )) + } + } + } +} diff --git a/crates/nu-derive-value/src/from.rs b/crates/nu-derive-value/src/from.rs new file mode 100644 index 0000000000..783a22920e --- /dev/null +++ b/crates/nu-derive-value/src/from.rs @@ -0,0 +1,567 @@ +use convert_case::Casing; +use proc_macro2::TokenStream as TokenStream2; +use quote::{quote, ToTokens}; +use syn::{ + spanned::Spanned, Attribute, Data, DataEnum, DataStruct, DeriveInput, Fields, Generics, Ident, + Type, +}; + +use crate::attributes::{self, ContainerAttributes}; + +#[derive(Debug)] +pub struct FromValue; +type DeriveError = super::error::DeriveError; + +/// Inner implementation of the `#[derive(FromValue)]` macro for structs and enums. +/// +/// Uses `proc_macro2::TokenStream` for better testing support, unlike `proc_macro::TokenStream`. +/// +/// This function directs the `FromValue` trait derivation to the correct implementation based on +/// the input type: +/// - For structs: [`derive_struct_from_value`] +/// - For enums: [`derive_enum_from_value`] +/// - Unions are not supported and will return an error. +pub fn derive_from_value(input: TokenStream2) -> Result { + let input: DeriveInput = syn::parse2(input).map_err(DeriveError::Syn)?; + match input.data { + Data::Struct(data_struct) => Ok(derive_struct_from_value( + input.ident, + data_struct, + input.generics, + input.attrs, + )?), + Data::Enum(data_enum) => Ok(derive_enum_from_value( + input.ident, + data_enum, + input.generics, + input.attrs, + )?), + Data::Union(_) => Err(DeriveError::UnsupportedUnions), + } +} + +/// Implements the `#[derive(FromValue)]` macro for structs. +/// +/// This function ensures that the helper attribute is not used anywhere, as it is not supported for +/// structs. +/// Other than this, this function provides the impl signature for `FromValue`. +/// The implementation for `FromValue::from_value` is handled by [`struct_from_value`] and the +/// `FromValue::expected_type` is handled by [`struct_expected_type`]. +fn derive_struct_from_value( + ident: Ident, + data: DataStruct, + generics: Generics, + attrs: Vec, +) -> Result { + attributes::deny(&attrs)?; + attributes::deny_fields(&data.fields)?; + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + let from_value_impl = struct_from_value(&data); + let expected_type_impl = struct_expected_type(&data.fields); + Ok(quote! { + #[automatically_derived] + impl #impl_generics nu_protocol::FromValue for #ident #ty_generics #where_clause { + #from_value_impl + #expected_type_impl + } + }) +} + +/// Implements `FromValue::from_value` for structs. +/// +/// This function constructs the `from_value` function for structs. +/// The implementation is straightforward as most of the heavy lifting is handled by +/// `parse_value_via_fields`, and this function only needs to construct the signature around it. +/// +/// For structs with named fields, this constructs a large return type where each field +/// contains the implementation for that specific field. +/// In structs with unnamed fields, a [`VecDeque`](std::collections::VecDeque) is used to load each +/// field one after another, and the result is used to construct the tuple. +/// For unit structs, this only checks if the input value is `Value::Nothing`. +/// +/// # Examples +/// +/// These examples show what the macro would generate. +/// +/// Struct with named fields: +/// ```rust +/// #[derive(IntoValue)] +/// struct Pet { +/// name: String, +/// age: u8, +/// favorite_toy: Option, +/// } +/// +/// impl nu_protocol::FromValue for Pet { +/// fn from_value( +/// v: nu_protocol::Value +/// ) -> std::result::Result { +/// let span = v.span(); +/// let mut record = v.into_record()?; +/// std::result::Result::Ok(Pet { +/// name: ::from_value( +/// record +/// .remove("name") +/// .ok_or_else(|| nu_protocol::ShellError::CantFindColumn { +/// col_name: std::string::ToString::to_string("name"), +/// span: std::option::Option::None, +/// src_span: span +/// })?, +/// )?, +/// age: ::from_value( +/// record +/// .remove("age") +/// .ok_or_else(|| nu_protocol::ShellError::CantFindColumn { +/// col_name: std::string::ToString::to_string("age"), +/// span: std::option::Option::None, +/// src_span: span +/// })?, +/// )?, +/// favorite_toy: record +/// .remove("favorite_toy") +/// .map(|v| <#ty as nu_protocol::FromValue>::from_value(v)) +/// .transpose()? +/// .flatten(), +/// }) +/// } +/// } +/// ``` +/// +/// Struct with unnamed fields: +/// ```rust +/// #[derive(IntoValue)] +/// struct Color(u8, u8, u8); +/// +/// impl nu_protocol::FromValue for Color { +/// fn from_value( +/// v: nu_protocol::Value +/// ) -> std::result::Result { +/// let span = v.span(); +/// let list = v.into_list()?; +/// let mut deque: std::collections::VecDeque<_> = std::convert::From::from(list); +/// std::result::Result::Ok(Self( +/// { +/// ::from_value( +/// deque +/// .pop_front() +/// .ok_or_else(|| nu_protocol::ShellError::CantFindColumn { +/// col_name: std::string::ToString::to_string(&0), +/// span: std::option::Option::None, +/// src_span: span +/// })?, +/// )? +/// }, +/// { +/// ::from_value( +/// deque +/// .pop_front() +/// .ok_or_else(|| nu_protocol::ShellError::CantFindColumn { +/// col_name: std::string::ToString::to_string(&1), +/// span: std::option::Option::None, +/// src_span: span +/// })?, +/// )? +/// }, +/// { +/// ::from_value( +/// deque +/// .pop_front() +/// .ok_or_else(|| nu_protocol::ShellError::CantFindColumn { +/// col_name: std::string::ToString::to_string(&2), +/// span: std::option::Option::None, +/// src_span: span +/// })?, +/// )? +/// } +/// )) +/// } +/// } +/// ``` +/// +/// Unit struct: +/// ```rust +/// #[derive(IntoValue)] +/// struct Unicorn; +/// +/// impl nu_protocol::FromValue for Unicorn { +/// fn from_value( +/// v: nu_protocol::Value +/// ) -> std::result::Result { +/// match v { +/// nu_protocol::Value::Nothing {..} => Ok(Self), +/// v => std::result::Result::Err(nu_protocol::ShellError::CantConvert { +/// to_type: std::string::ToString::to_string(&::expected_type()), +/// from_type: std::string::ToString::to_string(&v.get_type()), +/// span: v.span(), +/// help: std::option::Option::None +/// }) +/// } +/// } +/// } +/// ``` +fn struct_from_value(data: &DataStruct) -> TokenStream2 { + let body = parse_value_via_fields(&data.fields, quote!(Self)); + quote! { + fn from_value( + v: nu_protocol::Value + ) -> std::result::Result { + #body + } + } +} + +/// Implements `FromValue::expected_type` for structs. +/// +/// This function constructs the `expected_type` function for structs. +/// The type depends on the `fields`: named fields construct a record type with every key and type +/// laid out. +/// Unnamed fields construct a custom type with the name in the format like +/// `list[type0, type1, type2]`. +/// No fields expect the `Type::Nothing`. +/// +/// # Examples +/// +/// These examples show what the macro would generate. +/// +/// Struct with named fields: +/// ```rust +/// #[derive(IntoValue)] +/// struct Pet { +/// name: String, +/// age: u8, +/// favorite_toy: Option, +/// } +/// +/// impl nu_protocol::FromValue for Pet { +/// fn expected_type() -> nu_protocol::Type { +/// nu_protocol::Type::Record( +/// std::vec![ +/// ( +/// std::string::ToString::to_string("name"), +/// ::expected_type(), +/// ), +/// ( +/// std::string::ToString::to_string("age"), +/// ::expected_type(), +/// ), +/// ( +/// std::string::ToString::to_string("favorite_toy"), +/// as nu_protocol::FromValue>::expected_type(), +/// ) +/// ].into_boxed_slice() +/// ) +/// } +/// } +/// ``` +/// +/// Struct with unnamed fields: +/// ```rust +/// #[derive(IntoValue)] +/// struct Color(u8, u8, u8); +/// +/// impl nu_protocol::FromValue for Color { +/// fn expected_type() -> nu_protocol::Type { +/// nu_protocol::Type::Custom( +/// std::format!( +/// "[{}, {}, {}]", +/// ::expected_type(), +/// ::expected_type(), +/// ::expected_type() +/// ) +/// .into_boxed_str() +/// ) +/// } +/// } +/// ``` +/// +/// Unit struct: +/// ```rust +/// #[derive(IntoValue)] +/// struct Unicorn; +/// +/// impl nu_protocol::FromValue for Color { +/// fn expected_type() -> nu_protocol::Type { +/// nu_protocol::Type::Nothing +/// } +/// } +/// ``` +fn struct_expected_type(fields: &Fields) -> TokenStream2 { + let ty = match fields { + Fields::Named(fields) => { + let fields = fields.named.iter().map(|field| { + let ident = field.ident.as_ref().expect("named has idents"); + let ident_s = ident.to_string(); + let ty = &field.ty; + quote! {( + std::string::ToString::to_string(#ident_s), + <#ty as nu_protocol::FromValue>::expected_type(), + )} + }); + quote!(nu_protocol::Type::Record( + std::vec![#(#fields),*].into_boxed_slice() + )) + } + Fields::Unnamed(fields) => { + let mut iter = fields.unnamed.iter(); + let fields = fields.unnamed.iter().map(|field| { + let ty = &field.ty; + quote!(<#ty as nu_protocol::FromValue>::expected_type()) + }); + let mut template = String::new(); + template.push('['); + if iter.next().is_some() { + template.push_str("{}") + } + iter.for_each(|_| template.push_str(", {}")); + template.push(']'); + quote! { + nu_protocol::Type::Custom( + std::format!( + #template, + #(#fields),* + ) + .into_boxed_str() + ) + } + } + Fields::Unit => quote!(nu_protocol::Type::Nothing), + }; + + quote! { + fn expected_type() -> nu_protocol::Type { + #ty + } + } +} + +/// Implements the `#[derive(FromValue)]` macro for enums. +/// +/// This function constructs the implementation of the `FromValue` trait for enums. +/// It is designed to be on the same level as [`derive_struct_from_value`], even though this +/// function only provides the impl signature for `FromValue`. +/// The actual implementation for `FromValue::from_value` is handled by [`enum_from_value`]. +/// +/// Since variants are difficult to type with the current type system, this function uses the +/// default implementation for `expected_type`. +fn derive_enum_from_value( + ident: Ident, + data: DataEnum, + generics: Generics, + attrs: Vec, +) -> Result { + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + let from_value_impl = enum_from_value(&data, &attrs)?; + Ok(quote! { + #[automatically_derived] + impl #impl_generics nu_protocol::FromValue for #ident #ty_generics #where_clause { + #from_value_impl + } + }) +} + +/// Implements `FromValue::from_value` for enums. +/// +/// This function constructs the `from_value` implementation for enums. +/// It only accepts enums with unit variants, as it is currently unclear how other types of enums +/// should be represented via a `Value`. +/// This function checks that every field is a unit variant and constructs a match statement over +/// all possible variants. +/// The input value is expected to be a `Value::String` containing the name of the variant formatted +/// as defined by the `#[nu_value(rename_all = "...")]` attribute. +/// If no attribute is given, [`convert_case::Case::Snake`] is expected. +/// +/// If no matching variant is found, `ShellError::CantConvert` is returned. +/// +/// This is how such a derived implementation looks: +/// ```rust +/// #[derive(IntoValue)] +/// enum Weather { +/// Sunny, +/// Cloudy, +/// Raining +/// } +/// +/// impl nu_protocol::IntoValue for Weather { +/// fn into_value(self, span: nu_protocol::Span) -> nu_protocol::Value { +/// let span = v.span(); +/// let ty = v.get_type(); +/// +/// let s = v.into_string()?; +/// match s.as_str() { +/// "sunny" => std::result::Ok(Self::Sunny), +/// "cloudy" => std::result::Ok(Self::Cloudy), +/// "raining" => std::result::Ok(Self::Raining), +/// _ => std::result::Result::Err(nu_protocol::ShellError::CantConvert { +/// to_type: std::string::ToString::to_string( +/// &::expected_type() +/// ), +/// from_type: std::string::ToString::to_string(&ty), +/// span: span,help: std::option::Option::None, +/// }), +/// } +/// } +/// } +/// ``` +fn enum_from_value(data: &DataEnum, attrs: &[Attribute]) -> Result { + let container_attrs = ContainerAttributes::parse_attrs(attrs.iter())?; + let arms: Vec = data + .variants + .iter() + .map(|variant| { + attributes::deny(&variant.attrs)?; + let ident = &variant.ident; + let ident_s = format!("{ident}") + .as_str() + .to_case(container_attrs.rename_all); + match &variant.fields { + Fields::Named(fields) => Err(DeriveError::UnsupportedEnums { + fields_span: fields.span(), + }), + Fields::Unnamed(fields) => Err(DeriveError::UnsupportedEnums { + fields_span: fields.span(), + }), + Fields::Unit => Ok(quote!(#ident_s => std::result::Result::Ok(Self::#ident))), + } + }) + .collect::>()?; + + Ok(quote! { + fn from_value( + v: nu_protocol::Value + ) -> std::result::Result { + let span = v.span(); + let ty = v.get_type(); + + let s = v.into_string()?; + match s.as_str() { + #(#arms,)* + _ => std::result::Result::Err(nu_protocol::ShellError::CantConvert { + to_type: std::string::ToString::to_string( + &::expected_type() + ), + from_type: std::string::ToString::to_string(&ty), + span: span, + help: std::option::Option::None, + }), + } + } + }) +} + +/// Parses a `Value` into self. +/// +/// This function handles the actual parsing of a `Value` into self. +/// It takes two parameters: `fields` and `self_ident`. +/// The `fields` parameter determines the expected type of `Value`: named fields expect a +/// `Value::Record`, unnamed fields expect a `Value::List`, and a unit expects `Value::Nothing`. +/// +/// For named fields, the `fields` parameter indicates which field in the record corresponds to +/// which struct field. +/// For both named and unnamed fields, it also helps cast the type into a `FromValue` type. +/// This approach maintains +/// [hygiene](https://doc.rust-lang.org/reference/macros-by-example.html#hygiene). +/// +/// The `self_ident` parameter is used to describe the identifier of the returned value. +/// For structs, `Self` is usually sufficient, but for enums, `Self::Variant` may be needed in the +/// future. +/// +/// This function is more complex than the equivalent for `IntoValue` due to error handling +/// requirements. +/// For missing fields, `ShellError::CantFindColumn` is used, and for unit structs, +/// `ShellError::CantConvert` is used. +/// The implementation avoids local variables for fields to prevent accidental shadowing, ensuring +/// that poorly named fields don't cause issues. +/// While this style is not typically recommended in handwritten Rust, it is acceptable for code +/// generation. +fn parse_value_via_fields(fields: &Fields, self_ident: impl ToTokens) -> TokenStream2 { + match fields { + Fields::Named(fields) => { + let fields = fields.named.iter().map(|field| { + let ident = field.ident.as_ref().expect("named has idents"); + let ident_s = ident.to_string(); + let ty = &field.ty; + match type_is_option(ty) { + true => quote! { + #ident: record + .remove(#ident_s) + .map(|v| <#ty as nu_protocol::FromValue>::from_value(v)) + .transpose()? + .flatten() + }, + + false => quote! { + #ident: <#ty as nu_protocol::FromValue>::from_value( + record + .remove(#ident_s) + .ok_or_else(|| nu_protocol::ShellError::CantFindColumn { + col_name: std::string::ToString::to_string(#ident_s), + span: std::option::Option::None, + src_span: span + })?, + )? + }, + } + }); + quote! { + let span = v.span(); + let mut record = v.into_record()?; + std::result::Result::Ok(#self_ident {#(#fields),*}) + } + } + Fields::Unnamed(fields) => { + let fields = fields.unnamed.iter().enumerate().map(|(i, field)| { + let ty = &field.ty; + quote! {{ + <#ty as nu_protocol::FromValue>::from_value( + deque + .pop_front() + .ok_or_else(|| nu_protocol::ShellError::CantFindColumn { + col_name: std::string::ToString::to_string(&#i), + span: std::option::Option::None, + src_span: span + })?, + )? + }} + }); + quote! { + let span = v.span(); + let list = v.into_list()?; + let mut deque: std::collections::VecDeque<_> = std::convert::From::from(list); + std::result::Result::Ok(#self_ident(#(#fields),*)) + } + } + Fields::Unit => quote! { + match v { + nu_protocol::Value::Nothing {..} => Ok(#self_ident), + v => std::result::Result::Err(nu_protocol::ShellError::CantConvert { + to_type: std::string::ToString::to_string(&::expected_type()), + from_type: std::string::ToString::to_string(&v.get_type()), + span: v.span(), + help: std::option::Option::None + }) + } + }, + } +} + +const FULLY_QUALIFIED_OPTION: &str = "std::option::Option"; +const PARTIALLY_QUALIFIED_OPTION: &str = "option::Option"; +const PRELUDE_OPTION: &str = "Option"; + +/// Check if the field type is an `Option`. +/// +/// This function checks if a given type is an `Option`. +/// We assume that an `Option` is [`std::option::Option`] because we can't see the whole code and +/// can't ask the compiler itself. +/// If the `Option` type isn't `std::option::Option`, the user will get a compile error due to a +/// type mismatch. +/// It's very unusual for people to override `Option`, so this should rarely be an issue. +/// +/// When [rust#63084](https://github.com/rust-lang/rust/issues/63084) is resolved, we can use +/// [`std::any::type_name`] for a static assertion check to get a more direct error messages. +fn type_is_option(ty: &Type) -> bool { + let s = ty.to_token_stream().to_string(); + s.starts_with(PRELUDE_OPTION) + || s.starts_with(PARTIALLY_QUALIFIED_OPTION) + || s.starts_with(FULLY_QUALIFIED_OPTION) +} diff --git a/crates/nu-derive-value/src/into.rs b/crates/nu-derive-value/src/into.rs new file mode 100644 index 0000000000..a7cec06378 --- /dev/null +++ b/crates/nu-derive-value/src/into.rs @@ -0,0 +1,266 @@ +use convert_case::Casing; +use proc_macro2::TokenStream as TokenStream2; +use quote::{quote, ToTokens}; +use syn::{ + spanned::Spanned, Attribute, Data, DataEnum, DataStruct, DeriveInput, Fields, Generics, Ident, + Index, +}; + +use crate::attributes::{self, ContainerAttributes}; + +#[derive(Debug)] +pub struct IntoValue; +type DeriveError = super::error::DeriveError; + +/// Inner implementation of the `#[derive(IntoValue)]` macro for structs and enums. +/// +/// Uses `proc_macro2::TokenStream` for better testing support, unlike `proc_macro::TokenStream`. +/// +/// This function directs the `IntoValue` trait derivation to the correct implementation based on +/// the input type: +/// - For structs: [`struct_into_value`] +/// - For enums: [`enum_into_value`] +/// - Unions are not supported and will return an error. +pub fn derive_into_value(input: TokenStream2) -> Result { + let input: DeriveInput = syn::parse2(input).map_err(DeriveError::Syn)?; + match input.data { + Data::Struct(data_struct) => Ok(struct_into_value( + input.ident, + data_struct, + input.generics, + input.attrs, + )?), + Data::Enum(data_enum) => Ok(enum_into_value( + input.ident, + data_enum, + input.generics, + input.attrs, + )?), + Data::Union(_) => Err(DeriveError::UnsupportedUnions), + } +} + +/// Implements the `#[derive(IntoValue)]` macro for structs. +/// +/// Automatically derives the `IntoValue` trait for any struct where each field implements +/// `IntoValue`. +/// For structs with named fields, the derived implementation creates a `Value::Record` using the +/// struct fields as keys. +/// Each field value is converted using the `IntoValue::into_value` method. +/// For structs with unnamed fields, this generates a `Value::List` with each field in the list. +/// For unit structs, this generates `Value::Nothing`, because there is no data. +/// +/// Note: The helper attribute `#[nu_value(...)]` is currently not allowed on structs. +/// +/// # Examples +/// +/// These examples show what the macro would generate. +/// +/// Struct with named fields: +/// ```rust +/// #[derive(IntoValue)] +/// struct Pet { +/// name: String, +/// age: u8, +/// favorite_toy: Option, +/// } +/// +/// impl nu_protocol::IntoValue for Pet { +/// fn into_value(self, span: nu_protocol::Span) -> nu_protocol::Value { +/// nu_protocol::Value::record(nu_protocol::record! { +/// "name" => nu_protocol::IntoValue::into_value(self.name, span), +/// "age" => nu_protocol::IntoValue::into_value(self.age, span), +/// "favorite_toy" => nu_protocol::IntoValue::into_value(self.favorite_toy, span), +/// }, span) +/// } +/// } +/// ``` +/// +/// Struct with unnamed fields: +/// ```rust +/// #[derive(IntoValue)] +/// struct Color(u8, u8, u8); +/// +/// impl nu_protocol::IntoValue for Color { +/// fn into_value(self, span: nu_protocol::Span) -> nu_protocol::Value { +/// nu_protocol::Value::list(vec![ +/// nu_protocol::IntoValue::into_value(self.0, span), +/// nu_protocol::IntoValue::into_value(self.1, span), +/// nu_protocol::IntoValue::into_value(self.2, span), +/// ], span) +/// } +/// } +/// ``` +/// +/// Unit struct: +/// ```rust +/// #[derive(IntoValue)] +/// struct Unicorn; +/// +/// impl nu_protocol::IntoValue for Unicorn { +/// fn into_value(self, span: nu_protocol::Span) -> nu_protocol::Value { +/// nu_protocol::Value::nothing(span) +/// } +/// } +/// ``` +fn struct_into_value( + ident: Ident, + data: DataStruct, + generics: Generics, + attrs: Vec, +) -> Result { + attributes::deny(&attrs)?; + attributes::deny_fields(&data.fields)?; + let record = match &data.fields { + Fields::Named(fields) => { + let accessor = fields + .named + .iter() + .map(|field| field.ident.as_ref().expect("named has idents")) + .map(|ident| quote!(self.#ident)); + fields_return_value(&data.fields, accessor) + } + Fields::Unnamed(fields) => { + let accessor = fields + .unnamed + .iter() + .enumerate() + .map(|(n, _)| Index::from(n)) + .map(|index| quote!(self.#index)); + fields_return_value(&data.fields, accessor) + } + Fields::Unit => quote!(nu_protocol::Value::nothing(span)), + }; + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + Ok(quote! { + #[automatically_derived] + impl #impl_generics nu_protocol::IntoValue for #ident #ty_generics #where_clause { + fn into_value(self, span: nu_protocol::Span) -> nu_protocol::Value { + #record + } + } + }) +} + +/// Implements the `#[derive(IntoValue)]` macro for enums. +/// +/// This function implements the derive macro `IntoValue` for enums. +/// Currently, only unit enum variants are supported as it is not clear how other types of enums +/// should be represented in a `Value`. +/// For simple enums, we represent the enum as a `Value::String`. For other types of variants, we return an error. +/// The variant name will be case-converted as described by the `#[nu_value(rename_all = "...")]` helper attribute. +/// If no attribute is used, the default is `case_convert::Case::Snake`. +/// The implementation matches over all variants, uses the appropriate variant name, and constructs a `Value::String`. +/// +/// This is how such a derived implementation looks: +/// ```rust +/// #[derive(IntoValue)] +/// enum Weather { +/// Sunny, +/// Cloudy, +/// Raining +/// } +/// +/// impl nu_protocol::IntoValue for Weather { +/// fn into_value(self, span: nu_protocol::Span) -> nu_protocol::Value { +/// match self { +/// Self::Sunny => nu_protocol::Value::string("sunny", span), +/// Self::Cloudy => nu_protocol::Value::string("cloudy", span), +/// Self::Raining => nu_protocol::Value::string("raining", span), +/// } +/// } +/// } +/// ``` +fn enum_into_value( + ident: Ident, + data: DataEnum, + generics: Generics, + attrs: Vec, +) -> Result { + let container_attrs = ContainerAttributes::parse_attrs(attrs.iter())?; + let arms: Vec = data + .variants + .into_iter() + .map(|variant| { + attributes::deny(&variant.attrs)?; + let ident = variant.ident; + let ident_s = format!("{ident}") + .as_str() + .to_case(container_attrs.rename_all); + match &variant.fields { + // In the future we can implement more complexe enums here. + Fields::Named(fields) => Err(DeriveError::UnsupportedEnums { + fields_span: fields.span(), + }), + Fields::Unnamed(fields) => Err(DeriveError::UnsupportedEnums { + fields_span: fields.span(), + }), + Fields::Unit => { + Ok(quote!(Self::#ident => nu_protocol::Value::string(#ident_s, span))) + } + } + }) + .collect::>()?; + + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + Ok(quote! { + impl #impl_generics nu_protocol::IntoValue for #ident #ty_generics #where_clause { + fn into_value(self, span: nu_protocol::Span) -> nu_protocol::Value { + match self { + #(#arms,)* + } + } + } + }) +} + +/// Constructs the final `Value` that the macro generates. +/// +/// This function handles the construction of the final `Value` that the macro generates. +/// It is currently only used for structs but may be used for enums in the future. +/// The function takes two parameters: the `fields`, which allow iterating over each field of a data +/// type, and the `accessor`. +/// The fields determine whether we need to generate a `Value::Record`, `Value::List`, or +/// `Value::Nothing`. +/// For named fields, they are also directly used to generate the record key. +/// +/// The `accessor` parameter generalizes how the data is accessed. +/// For named fields, this is usually the name of the fields preceded by `self` in a struct, and +/// maybe something else for enums. +/// For unnamed fields, this should be an iterator similar to the one with named fields, but +/// accessing tuple fields, so we get `self.n`. +/// For unit structs, this parameter is ignored. +/// By using the accessor like this, we can have the same code for structs and enums with data +/// variants in the future. +fn fields_return_value( + fields: &Fields, + accessor: impl Iterator, +) -> TokenStream2 { + match fields { + Fields::Named(fields) => { + let items: Vec = fields + .named + .iter() + .zip(accessor) + .map(|(field, accessor)| { + let ident = field.ident.as_ref().expect("named has idents"); + let field = ident.to_string(); + quote!(#field => nu_protocol::IntoValue::into_value(#accessor, span)) + }) + .collect(); + quote! { + nu_protocol::Value::record(nu_protocol::record! { + #(#items),* + }, span) + } + } + Fields::Unnamed(fields) => { + let items = + fields.unnamed.iter().zip(accessor).map( + |(_, accessor)| quote!(nu_protocol::IntoValue::into_value(#accessor, span)), + ); + quote!(nu_protocol::Value::list(std::vec![#(#items),*], span)) + } + Fields::Unit => quote!(nu_protocol::Value::nothing(span)), + } +} diff --git a/crates/nu-derive-value/src/lib.rs b/crates/nu-derive-value/src/lib.rs new file mode 100644 index 0000000000..269566fc88 --- /dev/null +++ b/crates/nu-derive-value/src/lib.rs @@ -0,0 +1,69 @@ +//! Macro implementations of `#[derive(FromValue, IntoValue)]`. +//! +//! As this crate is a [`proc_macro`] crate, it is only allowed to export +//! [procedural macros](https://doc.rust-lang.org/reference/procedural-macros.html). +//! Therefore, it only exports [`IntoValue`] and [`FromValue`]. +//! +//! To get documentation for other functions and types used in this crate, run +//! `cargo doc -p nu-derive-value --document-private-items`. +//! +//! This crate uses a lot of +//! [`proc_macro2::TokenStream`](https://docs.rs/proc-macro2/1.0.24/proc_macro2/struct.TokenStream.html) +//! as `TokenStream2` to allow testing the behavior of the macros directly, including the output +//! token stream or if the macro errors as expected. +//! The tests for functionality can be found in `nu_protocol::value::test_derive`. +//! +//! This documentation is often less reference-heavy than typical Rust documentation. +//! This is because this crate is a dependency for `nu_protocol`, and linking to it would create a +//! cyclic dependency. +//! Also all examples in the documentation aren't tested as this crate cannot be compiled as a +//! normal library very easily. +//! This might change in the future if cargo allows building a proc-macro crate differently for +//! `cfg(doctest)` as they are already doing for `cfg(test)`. +//! +//! The generated code from the derive macros tries to be as +//! [hygienic](https://doc.rust-lang.org/reference/macros-by-example.html#hygiene) as possible. +//! This ensures that the macro can be called anywhere without requiring specific imports. +//! This results in obtuse code, which isn't recommended for manual, handwritten Rust +//! but ensures that no other code may influence this generated code or vice versa. + +use proc_macro::TokenStream; +use proc_macro2::TokenStream as TokenStream2; +use proc_macro_error::{proc_macro_error, Diagnostic}; + +mod attributes; +mod error; +mod from; +mod into; +#[cfg(test)] +mod tests; + +const HELPER_ATTRIBUTE: &str = "nu_value"; + +/// Derive macro generating an impl of the trait `IntoValue`. +/// +/// For further information, see the docs on the trait itself. +#[proc_macro_derive(IntoValue, attributes(nu_value))] +#[proc_macro_error] +pub fn derive_into_value(input: TokenStream) -> TokenStream { + let input = TokenStream2::from(input); + let output = match into::derive_into_value(input) { + Ok(output) => output, + Err(e) => Diagnostic::from(e).abort(), + }; + TokenStream::from(output) +} + +/// Derive macro generating an impl of the trait `FromValue`. +/// +/// For further information, see the docs on the trait itself. +#[proc_macro_derive(FromValue, attributes(nu_value))] +#[proc_macro_error] +pub fn derive_from_value(input: TokenStream) -> TokenStream { + let input = TokenStream2::from(input); + let output = match from::derive_from_value(input) { + Ok(output) => output, + Err(e) => Diagnostic::from(e).abort(), + }; + TokenStream::from(output) +} diff --git a/crates/nu-derive-value/src/tests.rs b/crates/nu-derive-value/src/tests.rs new file mode 100644 index 0000000000..b94d12a732 --- /dev/null +++ b/crates/nu-derive-value/src/tests.rs @@ -0,0 +1,157 @@ +// These tests only check that the derive macros throw the relevant errors. +// Functionality of the derived types is tested in nu_protocol::value::test_derive. + +use crate::error::DeriveError; +use crate::from::derive_from_value; +use crate::into::derive_into_value; +use quote::quote; + +#[test] +fn unsupported_unions() { + let input = quote! { + #[nu_value] + union SomeUnion { + f1: u32, + f2: f32, + } + }; + + let from_res = derive_from_value(input.clone()); + assert!( + matches!(from_res, Err(DeriveError::UnsupportedUnions)), + "expected `DeriveError::UnsupportedUnions`, got {:?}", + from_res + ); + + let into_res = derive_into_value(input); + assert!( + matches!(into_res, Err(DeriveError::UnsupportedUnions)), + "expected `DeriveError::UnsupportedUnions`, got {:?}", + into_res + ); +} + +#[test] +fn unsupported_enums() { + let input = quote! { + #[nu_value(rename_all = "SCREAMING_SNAKE_CASE")] + enum ComplexEnum { + Unit, + Unnamed(u32, f32), + Named { + u: u32, + f: f32, + } + } + }; + + let from_res = derive_from_value(input.clone()); + assert!( + matches!(from_res, Err(DeriveError::UnsupportedEnums { .. })), + "expected `DeriveError::UnsupportedEnums`, got {:?}", + from_res + ); + + let into_res = derive_into_value(input); + assert!( + matches!(into_res, Err(DeriveError::UnsupportedEnums { .. })), + "expected `DeriveError::UnsupportedEnums`, got {:?}", + into_res + ); +} + +#[test] +fn unexpected_attribute() { + let input = quote! { + #[nu_value(what)] + enum SimpleEnum { + A, + B, + } + }; + + let from_res = derive_from_value(input.clone()); + assert!( + matches!(from_res, Err(DeriveError::UnexpectedAttribute { .. })), + "expected `DeriveError::UnexpectedAttribute`, got {:?}", + from_res + ); + + let into_res = derive_into_value(input); + assert!( + matches!(into_res, Err(DeriveError::UnexpectedAttribute { .. })), + "expected `DeriveError::UnexpectedAttribute`, got {:?}", + into_res + ); +} + +#[test] +fn deny_attribute_on_structs() { + let input = quote! { + #[nu_value] + struct SomeStruct; + }; + + let from_res = derive_from_value(input.clone()); + assert!( + matches!(from_res, Err(DeriveError::InvalidAttributePosition { .. })), + "expected `DeriveError::InvalidAttributePosition`, got {:?}", + from_res + ); + + let into_res = derive_into_value(input); + assert!( + matches!(into_res, Err(DeriveError::InvalidAttributePosition { .. })), + "expected `DeriveError::InvalidAttributePosition`, got {:?}", + into_res + ); +} + +#[test] +fn deny_attribute_on_fields() { + let input = quote! { + struct SomeStruct { + #[nu_value] + field: () + } + }; + + let from_res = derive_from_value(input.clone()); + assert!( + matches!(from_res, Err(DeriveError::InvalidAttributePosition { .. })), + "expected `DeriveError::InvalidAttributePosition`, got {:?}", + from_res + ); + + let into_res = derive_into_value(input); + assert!( + matches!(into_res, Err(DeriveError::InvalidAttributePosition { .. })), + "expected `DeriveError::InvalidAttributePosition`, got {:?}", + into_res + ); +} + +#[test] +fn invalid_attribute_value() { + let input = quote! { + #[nu_value(rename_all = "CrazY-CasE")] + enum SimpleEnum { + A, + B + } + }; + + let from_res = derive_from_value(input.clone()); + assert!( + matches!(from_res, Err(DeriveError::InvalidAttributeValue { .. })), + "expected `DeriveError::InvalidAttributeValue`, got {:?}", + from_res + ); + + let into_res = derive_into_value(input); + assert!( + matches!(into_res, Err(DeriveError::InvalidAttributeValue { .. })), + "expected `DeriveError::InvalidAttributeValue`, got {:?}", + into_res + ); +} diff --git a/crates/nu-engine/Cargo.toml b/crates/nu-engine/Cargo.toml index 39fefc525e..c416a2f590 100644 --- a/crates/nu-engine/Cargo.toml +++ b/crates/nu-engine/Cargo.toml @@ -5,16 +5,17 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-engine" edition = "2021" license = "MIT" name = "nu-engine" -version = "0.93.1" +version = "0.95.1" [lib] bench = false [dependencies] -nu-protocol = { path = "../nu-protocol", features = ["plugin"], version = "0.93.1" } -nu-path = { path = "../nu-path", version = "0.93.1" } -nu-glob = { path = "../nu-glob", version = "0.93.1" } -nu-utils = { path = "../nu-utils", version = "0.93.1" } +nu-protocol = { path = "../nu-protocol", features = ["plugin"], version = "0.95.1" } +nu-path = { path = "../nu-path", version = "0.95.1" } +nu-glob = { path = "../nu-glob", version = "0.95.1" } +nu-utils = { path = "../nu-utils", version = "0.95.1" } +log = { workspace = true } [features] plugin = [] diff --git a/crates/nu-engine/src/call_ext.rs b/crates/nu-engine/src/call_ext.rs index daedb24a1f..d3f36215e6 100644 --- a/crates/nu-engine/src/call_ext.rs +++ b/crates/nu-engine/src/call_ext.rs @@ -1,10 +1,10 @@ use crate::eval_expression; use nu_protocol::{ - ast::Call, + ast, debugger::WithoutDebug, - engine::{EngineState, Stack, StateWorkingSet}, + engine::{self, EngineState, Stack, StateWorkingSet}, eval_const::eval_constant, - FromValue, ShellError, Value, + ir, FromValue, ShellError, Span, Value, }; pub trait CallExt { @@ -23,6 +23,9 @@ pub trait CallExt { name: &str, ) -> Result, ShellError>; + /// Efficiently get the span of a flag argument + fn get_flag_span(&self, stack: &Stack, name: &str) -> Option; + fn rest( &self, engine_state: &EngineState, @@ -56,9 +59,12 @@ pub trait CallExt { stack: &mut Stack, name: &str, ) -> Result; + + /// True if the command has any positional or rest arguments, excluding before the given index. + fn has_positional_args(&self, stack: &Stack, starting_pos: usize) -> bool; } -impl CallExt for Call { +impl CallExt for ast::Call { fn has_flag( &self, engine_state: &EngineState, @@ -104,6 +110,10 @@ impl CallExt for Call { } } + fn get_flag_span(&self, _stack: &Stack, name: &str) -> Option { + self.get_named_arg(name).map(|arg| arg.span) + } + fn rest( &self, engine_state: &EngineState, @@ -189,4 +199,205 @@ impl CallExt for Call { }) } } + + fn has_positional_args(&self, _stack: &Stack, starting_pos: usize) -> bool { + self.rest_iter(starting_pos).next().is_some() + } +} + +impl CallExt for ir::Call { + fn has_flag( + &self, + _engine_state: &EngineState, + stack: &mut Stack, + flag_name: &str, + ) -> Result { + Ok(self + .named_iter(stack) + .find(|(name, _)| name.item == flag_name) + .is_some_and(|(_, value)| { + // Handle --flag=false + !matches!(value, Some(Value::Bool { val: false, .. })) + })) + } + + fn get_flag( + &self, + _engine_state: &EngineState, + stack: &mut Stack, + name: &str, + ) -> Result, ShellError> { + if let Some(val) = self.get_named_arg(stack, name) { + T::from_value(val.clone()).map(Some) + } else { + Ok(None) + } + } + + fn get_flag_span(&self, stack: &Stack, name: &str) -> Option { + self.named_iter(stack) + .find_map(|(i_name, _)| (i_name.item == name).then_some(i_name.span)) + } + + fn rest( + &self, + _engine_state: &EngineState, + stack: &mut Stack, + starting_pos: usize, + ) -> Result, ShellError> { + self.rest_iter_flattened(stack, starting_pos)? + .into_iter() + .map(T::from_value) + .collect() + } + + fn opt( + &self, + _engine_state: &EngineState, + stack: &mut Stack, + pos: usize, + ) -> Result, ShellError> { + self.positional_iter(stack) + .nth(pos) + .cloned() + .map(T::from_value) + .transpose() + } + + fn opt_const( + &self, + _working_set: &StateWorkingSet, + _pos: usize, + ) -> Result, ShellError> { + Err(ShellError::IrEvalError { + msg: "const evaluation is not yet implemented on ir::Call".into(), + span: Some(self.head), + }) + } + + fn req( + &self, + engine_state: &EngineState, + stack: &mut Stack, + pos: usize, + ) -> Result { + if let Some(val) = self.opt(engine_state, stack, pos)? { + Ok(val) + } else if self.positional_len(stack) == 0 { + Err(ShellError::AccessEmptyContent { span: self.head }) + } else { + Err(ShellError::AccessBeyondEnd { + max_idx: self.positional_len(stack) - 1, + span: self.head, + }) + } + } + + fn req_parser_info( + &self, + engine_state: &EngineState, + stack: &mut Stack, + name: &str, + ) -> Result { + // FIXME: this depends on the AST evaluator. We can fix this by making the parser info an + // enum rather than using expressions. It's not clear that evaluation of this is ever really + // needed. + if let Some(expr) = self.get_parser_info(stack, name) { + let expr = expr.clone(); + let stack = &mut stack.use_call_arg_out_dest(); + let result = eval_expression::(engine_state, stack, &expr)?; + FromValue::from_value(result) + } else { + Err(ShellError::CantFindColumn { + col_name: name.into(), + span: None, + src_span: self.head, + }) + } + } + + fn has_positional_args(&self, stack: &Stack, starting_pos: usize) -> bool { + self.rest_iter(stack, starting_pos).next().is_some() + } +} + +macro_rules! proxy { + ($self:ident . $method:ident ($($param:expr),*)) => (match &$self.inner { + engine::CallImpl::AstRef(call) => call.$method($($param),*), + engine::CallImpl::AstBox(call) => call.$method($($param),*), + engine::CallImpl::IrRef(call) => call.$method($($param),*), + engine::CallImpl::IrBox(call) => call.$method($($param),*), + }) +} + +impl CallExt for engine::Call<'_> { + fn has_flag( + &self, + engine_state: &EngineState, + stack: &mut Stack, + flag_name: &str, + ) -> Result { + proxy!(self.has_flag(engine_state, stack, flag_name)) + } + + fn get_flag( + &self, + engine_state: &EngineState, + stack: &mut Stack, + name: &str, + ) -> Result, ShellError> { + proxy!(self.get_flag(engine_state, stack, name)) + } + + fn get_flag_span(&self, stack: &Stack, name: &str) -> Option { + proxy!(self.get_flag_span(stack, name)) + } + + fn rest( + &self, + engine_state: &EngineState, + stack: &mut Stack, + starting_pos: usize, + ) -> Result, ShellError> { + proxy!(self.rest(engine_state, stack, starting_pos)) + } + + fn opt( + &self, + engine_state: &EngineState, + stack: &mut Stack, + pos: usize, + ) -> Result, ShellError> { + proxy!(self.opt(engine_state, stack, pos)) + } + + fn opt_const( + &self, + working_set: &StateWorkingSet, + pos: usize, + ) -> Result, ShellError> { + proxy!(self.opt_const(working_set, pos)) + } + + fn req( + &self, + engine_state: &EngineState, + stack: &mut Stack, + pos: usize, + ) -> Result { + proxy!(self.req(engine_state, stack, pos)) + } + + fn req_parser_info( + &self, + engine_state: &EngineState, + stack: &mut Stack, + name: &str, + ) -> Result { + proxy!(self.req_parser_info(engine_state, stack, name)) + } + + fn has_positional_args(&self, stack: &Stack, starting_pos: usize) -> bool { + proxy!(self.has_positional_args(stack, starting_pos)) + } } diff --git a/crates/nu-engine/src/closure_eval.rs b/crates/nu-engine/src/closure_eval.rs index f4bc40658b..b271d90cbe 100644 --- a/crates/nu-engine/src/closure_eval.rs +++ b/crates/nu-engine/src/closure_eval.rs @@ -62,8 +62,8 @@ pub struct ClosureEval { stack: Stack, block: Arc, arg_index: usize, - env_vars: Vec, - env_hidden: HashMap>, + env_vars: Vec>, + env_hidden: Arc>>, eval: EvalBlockWithEarlyReturnFn, } diff --git a/crates/nu-engine/src/command_prelude.rs b/crates/nu-engine/src/command_prelude.rs index 089a2fb8fa..e6ddb5fb91 100644 --- a/crates/nu-engine/src/command_prelude.rs +++ b/crates/nu-engine/src/command_prelude.rs @@ -1,8 +1,8 @@ pub use crate::CallExt; pub use nu_protocol::{ - ast::{Call, CellPath}, - engine::{Command, EngineState, Stack}, - record, Category, ErrSpan, Example, IntoInterruptiblePipelineData, IntoPipelineData, - IntoSpanned, PipelineData, Record, ShellError, Signature, Span, Spanned, SyntaxShape, Type, - Value, + ast::CellPath, + engine::{Call, Command, EngineState, Stack, StateWorkingSet}, + record, ByteStream, ByteStreamType, Category, ErrSpan, Example, IntoInterruptiblePipelineData, + IntoPipelineData, IntoSpanned, PipelineData, Record, ShellError, Signature, Span, Spanned, + SyntaxShape, Type, Value, }; diff --git a/crates/nu-engine/src/compile/builder.rs b/crates/nu-engine/src/compile/builder.rs new file mode 100644 index 0000000000..d77075cc3f --- /dev/null +++ b/crates/nu-engine/src/compile/builder.rs @@ -0,0 +1,575 @@ +use nu_protocol::{ + ir::{DataSlice, Instruction, IrAstRef, IrBlock, Literal}, + CompileError, IntoSpanned, RegId, Span, Spanned, +}; + +/// A label identifier. Only exists while building code. Replaced with the actual target. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct LabelId(pub usize); + +/// Builds [`IrBlock`]s progressively by consuming instructions and handles register allocation. +#[derive(Debug)] +pub(crate) struct BlockBuilder { + pub(crate) block_span: Option, + pub(crate) instructions: Vec, + pub(crate) spans: Vec, + /// The actual instruction index that a label refers to. While building IR, branch targets are + /// specified as indices into this array rather than the true instruction index. This makes it + /// easier to make modifications to code, as just this array needs to be changed, and it's also + /// less error prone as during `finish()` we check to make sure all of the used labels have had + /// an index actually set. + pub(crate) labels: Vec>, + pub(crate) data: Vec, + pub(crate) ast: Vec>, + pub(crate) comments: Vec, + pub(crate) register_allocation_state: Vec, + pub(crate) file_count: u32, + pub(crate) loop_stack: Vec, +} + +impl BlockBuilder { + /// Starts a new block, with the first register (`%0`) allocated as input. + pub(crate) fn new(block_span: Option) -> Self { + BlockBuilder { + block_span, + instructions: vec![], + spans: vec![], + labels: vec![], + data: vec![], + ast: vec![], + comments: vec![], + register_allocation_state: vec![true], + file_count: 0, + loop_stack: vec![], + } + } + + /// Get the next unused register for code generation. + pub(crate) fn next_register(&mut self) -> Result { + if let Some(index) = self + .register_allocation_state + .iter_mut() + .position(|is_allocated| { + if !*is_allocated { + *is_allocated = true; + true + } else { + false + } + }) + { + Ok(RegId(index as u32)) + } else if self.register_allocation_state.len() < (u32::MAX as usize - 2) { + let reg_id = RegId(self.register_allocation_state.len() as u32); + self.register_allocation_state.push(true); + Ok(reg_id) + } else { + Err(CompileError::RegisterOverflow { + block_span: self.block_span, + }) + } + } + + /// Check if a register is initialized with a value. + pub(crate) fn is_allocated(&self, reg_id: RegId) -> bool { + self.register_allocation_state + .get(reg_id.0 as usize) + .is_some_and(|state| *state) + } + + /// Mark a register as initialized. + pub(crate) fn mark_register(&mut self, reg_id: RegId) -> Result<(), CompileError> { + if let Some(is_allocated) = self.register_allocation_state.get_mut(reg_id.0 as usize) { + *is_allocated = true; + Ok(()) + } else { + Err(CompileError::RegisterOverflow { + block_span: self.block_span, + }) + } + } + + /// Mark a register as empty, so that it can be used again by something else. + #[track_caller] + pub(crate) fn free_register(&mut self, reg_id: RegId) -> Result<(), CompileError> { + let index = reg_id.0 as usize; + + if self + .register_allocation_state + .get(index) + .is_some_and(|is_allocated| *is_allocated) + { + self.register_allocation_state[index] = false; + Ok(()) + } else { + log::warn!("register {reg_id} uninitialized, builder = {self:#?}"); + Err(CompileError::RegisterUninitialized { + reg_id, + caller: std::panic::Location::caller().to_string(), + }) + } + } + + /// Define a label, which can be used by branch instructions. The target can optionally be + /// specified now. + pub(crate) fn label(&mut self, target_index: Option) -> LabelId { + let label_id = self.labels.len(); + self.labels.push(target_index); + LabelId(label_id) + } + + /// Change the target of a label. + pub(crate) fn set_label( + &mut self, + label_id: LabelId, + target_index: usize, + ) -> Result<(), CompileError> { + *self + .labels + .get_mut(label_id.0) + .ok_or(CompileError::UndefinedLabel { + label_id: label_id.0, + span: None, + })? = Some(target_index); + Ok(()) + } + + /// Insert an instruction into the block, automatically marking any registers populated by + /// the instruction, and freeing any registers consumed by the instruction. + #[track_caller] + pub(crate) fn push(&mut self, instruction: Spanned) -> Result<(), CompileError> { + // Free read registers, and mark write registers. + // + // If a register is both read and written, it should be on both sides, so that we can verify + // that the register was in the right state beforehand. + let mut allocate = |read: &[RegId], write: &[RegId]| -> Result<(), CompileError> { + for reg in read { + self.free_register(*reg)?; + } + for reg in write { + self.mark_register(*reg)?; + } + Ok(()) + }; + + let allocate_result = match &instruction.item { + Instruction::Unreachable => Ok(()), + Instruction::LoadLiteral { dst, lit } => { + allocate(&[], &[*dst]).and( + // Free any registers on the literal + match lit { + Literal::Range { + start, + step, + end, + inclusion: _, + } => allocate(&[*start, *step, *end], &[]), + Literal::Bool(_) + | Literal::Int(_) + | Literal::Float(_) + | Literal::Filesize(_) + | Literal::Duration(_) + | Literal::Binary(_) + | Literal::Block(_) + | Literal::Closure(_) + | Literal::RowCondition(_) + | Literal::List { capacity: _ } + | Literal::Record { capacity: _ } + | Literal::Filepath { + val: _, + no_expand: _, + } + | Literal::Directory { + val: _, + no_expand: _, + } + | Literal::GlobPattern { + val: _, + no_expand: _, + } + | Literal::String(_) + | Literal::RawString(_) + | Literal::CellPath(_) + | Literal::Date(_) + | Literal::Nothing => Ok(()), + }, + ) + } + Instruction::LoadValue { dst, val: _ } => allocate(&[], &[*dst]), + Instruction::Move { dst, src } => allocate(&[*src], &[*dst]), + Instruction::Clone { dst, src } => allocate(&[*src], &[*dst, *src]), + Instruction::Collect { src_dst } => allocate(&[*src_dst], &[*src_dst]), + Instruction::Span { src_dst } => allocate(&[*src_dst], &[*src_dst]), + Instruction::Drop { src } => allocate(&[*src], &[]), + Instruction::Drain { src } => allocate(&[*src], &[]), + Instruction::LoadVariable { dst, var_id: _ } => allocate(&[], &[*dst]), + Instruction::StoreVariable { var_id: _, src } => allocate(&[*src], &[]), + Instruction::LoadEnv { dst, key: _ } => allocate(&[], &[*dst]), + Instruction::LoadEnvOpt { dst, key: _ } => allocate(&[], &[*dst]), + Instruction::StoreEnv { key: _, src } => allocate(&[*src], &[]), + Instruction::PushPositional { src } => allocate(&[*src], &[]), + Instruction::AppendRest { src } => allocate(&[*src], &[]), + Instruction::PushFlag { name: _ } => Ok(()), + Instruction::PushShortFlag { short: _ } => Ok(()), + Instruction::PushNamed { name: _, src } => allocate(&[*src], &[]), + Instruction::PushShortNamed { short: _, src } => allocate(&[*src], &[]), + Instruction::PushParserInfo { name: _, info: _ } => Ok(()), + Instruction::RedirectOut { mode: _ } => Ok(()), + Instruction::RedirectErr { mode: _ } => Ok(()), + Instruction::CheckErrRedirected { src } => allocate(&[*src], &[*src]), + Instruction::OpenFile { + file_num: _, + path, + append: _, + } => allocate(&[*path], &[]), + Instruction::WriteFile { file_num: _, src } => allocate(&[*src], &[]), + Instruction::CloseFile { file_num: _ } => Ok(()), + Instruction::Call { + decl_id: _, + src_dst, + } => allocate(&[*src_dst], &[*src_dst]), + Instruction::StringAppend { src_dst, val } => allocate(&[*src_dst, *val], &[*src_dst]), + Instruction::GlobFrom { + src_dst, + no_expand: _, + } => allocate(&[*src_dst], &[*src_dst]), + Instruction::ListPush { src_dst, item } => allocate(&[*src_dst, *item], &[*src_dst]), + Instruction::ListSpread { src_dst, items } => { + allocate(&[*src_dst, *items], &[*src_dst]) + } + Instruction::RecordInsert { src_dst, key, val } => { + allocate(&[*src_dst, *key, *val], &[*src_dst]) + } + Instruction::RecordSpread { src_dst, items } => { + allocate(&[*src_dst, *items], &[*src_dst]) + } + Instruction::Not { src_dst } => allocate(&[*src_dst], &[*src_dst]), + Instruction::BinaryOp { + lhs_dst, + op: _, + rhs, + } => allocate(&[*lhs_dst, *rhs], &[*lhs_dst]), + Instruction::FollowCellPath { src_dst, path } => { + allocate(&[*src_dst, *path], &[*src_dst]) + } + Instruction::CloneCellPath { dst, src, path } => { + allocate(&[*src, *path], &[*src, *dst]) + } + Instruction::UpsertCellPath { + src_dst, + path, + new_value, + } => allocate(&[*src_dst, *path, *new_value], &[*src_dst]), + Instruction::Jump { index: _ } => Ok(()), + Instruction::BranchIf { cond, index: _ } => allocate(&[*cond], &[]), + Instruction::BranchIfEmpty { src, index: _ } => allocate(&[*src], &[*src]), + Instruction::Match { + pattern: _, + src, + index: _, + } => allocate(&[*src], &[*src]), + Instruction::CheckMatchGuard { src } => allocate(&[*src], &[*src]), + Instruction::Iterate { + dst, + stream, + end_index: _, + } => allocate(&[*stream], &[*dst, *stream]), + Instruction::OnError { index: _ } => Ok(()), + Instruction::OnErrorInto { index: _, dst } => allocate(&[], &[*dst]), + Instruction::PopErrorHandler => Ok(()), + Instruction::CheckExternalFailed { dst, src } => allocate(&[*src], &[*dst, *src]), + Instruction::ReturnEarly { src } => allocate(&[*src], &[]), + Instruction::Return { src } => allocate(&[*src], &[]), + }; + + // Add more context to the error + match allocate_result { + Ok(()) => (), + Err(CompileError::RegisterUninitialized { reg_id, caller }) => { + return Err(CompileError::RegisterUninitializedWhilePushingInstruction { + reg_id, + caller, + instruction: format!("{:?}", instruction.item), + span: instruction.span, + }); + } + Err(err) => return Err(err), + } + + self.instructions.push(instruction.item); + self.spans.push(instruction.span); + self.ast.push(None); + self.comments.push(String::new()); + Ok(()) + } + + /// Set the AST of the last instruction. Separate method because it's rarely used. + pub(crate) fn set_last_ast(&mut self, ast_ref: Option) { + *self.ast.last_mut().expect("no last instruction") = ast_ref; + } + + /// Add a comment to the last instruction. + pub(crate) fn add_comment(&mut self, comment: impl std::fmt::Display) { + add_comment( + self.comments.last_mut().expect("no last instruction"), + comment, + ) + } + + /// Load a register with a literal. + pub(crate) fn load_literal( + &mut self, + reg_id: RegId, + literal: Spanned, + ) -> Result<(), CompileError> { + self.push( + Instruction::LoadLiteral { + dst: reg_id, + lit: literal.item, + } + .into_spanned(literal.span), + )?; + Ok(()) + } + + /// Allocate a new register and load a literal into it. + pub(crate) fn literal(&mut self, literal: Spanned) -> Result { + let reg_id = self.next_register()?; + self.load_literal(reg_id, literal)?; + Ok(reg_id) + } + + /// Deallocate a register and set it to `Empty`, if it is allocated + pub(crate) fn drop_reg(&mut self, reg_id: RegId) -> Result<(), CompileError> { + if self.is_allocated(reg_id) { + self.push(Instruction::Drop { src: reg_id }.into_spanned(Span::unknown()))?; + } + Ok(()) + } + + /// Set a register to `Empty`, but mark it as in-use, e.g. for input + pub(crate) fn load_empty(&mut self, reg_id: RegId) -> Result<(), CompileError> { + self.drop_reg(reg_id)?; + self.mark_register(reg_id) + } + + /// Drain the stream in a register (fully consuming it) + pub(crate) fn drain(&mut self, src: RegId, span: Span) -> Result<(), CompileError> { + self.push(Instruction::Drain { src }.into_spanned(span)) + } + + /// Add data to the `data` array and return a [`DataSlice`] referencing it. + pub(crate) fn data(&mut self, data: impl AsRef<[u8]>) -> Result { + let data = data.as_ref(); + let start = self.data.len(); + if data.is_empty() { + Ok(DataSlice::empty()) + } else if start + data.len() < u32::MAX as usize { + let slice = DataSlice { + start: start as u32, + len: data.len() as u32, + }; + self.data.extend_from_slice(data); + Ok(slice) + } else { + Err(CompileError::DataOverflow { + block_span: self.block_span, + }) + } + } + + /// Clone a register with a `clone` instruction. + pub(crate) fn clone_reg(&mut self, src: RegId, span: Span) -> Result { + let dst = self.next_register()?; + self.push(Instruction::Clone { dst, src }.into_spanned(span))?; + Ok(dst) + } + + /// Add a `branch-if` instruction + pub(crate) fn branch_if( + &mut self, + cond: RegId, + label_id: LabelId, + span: Span, + ) -> Result<(), CompileError> { + self.push( + Instruction::BranchIf { + cond, + index: label_id.0, + } + .into_spanned(span), + ) + } + + /// Add a `branch-if-empty` instruction + pub(crate) fn branch_if_empty( + &mut self, + src: RegId, + label_id: LabelId, + span: Span, + ) -> Result<(), CompileError> { + self.push( + Instruction::BranchIfEmpty { + src, + index: label_id.0, + } + .into_spanned(span), + ) + } + + /// Add a `jump` instruction + pub(crate) fn jump(&mut self, label_id: LabelId, span: Span) -> Result<(), CompileError> { + self.push(Instruction::Jump { index: label_id.0 }.into_spanned(span)) + } + + /// The index that the next instruction [`.push()`]ed will have. + pub(crate) fn here(&self) -> usize { + self.instructions.len() + } + + /// Allocate a new file number, for redirection. + pub(crate) fn next_file_num(&mut self) -> Result { + let next = self.file_count; + self.file_count = self + .file_count + .checked_add(1) + .ok_or(CompileError::FileOverflow { + block_span: self.block_span, + })?; + Ok(next) + } + + /// Push a new loop state onto the builder. Creates new labels that must be set. + pub(crate) fn begin_loop(&mut self) -> Loop { + let loop_ = Loop { + break_label: self.label(None), + continue_label: self.label(None), + }; + self.loop_stack.push(loop_); + loop_ + } + + /// True if we are currently in a loop. + pub(crate) fn is_in_loop(&self) -> bool { + !self.loop_stack.is_empty() + } + + /// Add a loop breaking jump instruction. + pub(crate) fn push_break(&mut self, span: Span) -> Result<(), CompileError> { + let loop_ = self + .loop_stack + .last() + .ok_or_else(|| CompileError::NotInALoop { + msg: "`break` called from outside of a loop".into(), + span: Some(span), + })?; + self.jump(loop_.break_label, span) + } + + /// Add a loop continuing jump instruction. + pub(crate) fn push_continue(&mut self, span: Span) -> Result<(), CompileError> { + let loop_ = self + .loop_stack + .last() + .ok_or_else(|| CompileError::NotInALoop { + msg: "`continue` called from outside of a loop".into(), + span: Some(span), + })?; + self.jump(loop_.continue_label, span) + } + + /// Pop the loop state. Checks that the loop being ended is the same one that was expected. + pub(crate) fn end_loop(&mut self, loop_: Loop) -> Result<(), CompileError> { + let ended_loop = self + .loop_stack + .pop() + .ok_or_else(|| CompileError::NotInALoop { + msg: "end_loop() called outside of a loop".into(), + span: None, + })?; + + if ended_loop == loop_ { + Ok(()) + } else { + Err(CompileError::IncoherentLoopState { + block_span: self.block_span, + }) + } + } + + /// Mark an unreachable code path. Produces an error at runtime if executed. + pub(crate) fn unreachable(&mut self, span: Span) -> Result<(), CompileError> { + self.push(Instruction::Unreachable.into_spanned(span)) + } + + /// Consume the builder and produce the final [`IrBlock`]. + pub(crate) fn finish(mut self) -> Result { + // Add comments to label targets + for (index, label_target) in self.labels.iter().enumerate() { + if let Some(label_target) = label_target { + add_comment( + &mut self.comments[*label_target], + format_args!("label({index})"), + ); + } + } + + // Populate the actual target indices of labels into the instructions + for ((index, instruction), span) in + self.instructions.iter_mut().enumerate().zip(&self.spans) + { + if let Some(label_id) = instruction.branch_target() { + let target_index = self.labels.get(label_id).cloned().flatten().ok_or( + CompileError::UndefinedLabel { + label_id, + span: Some(*span), + }, + )?; + // Add a comment to the target index that we come from here + add_comment( + &mut self.comments[target_index], + format_args!("from({index}:)"), + ); + instruction.set_branch_target(target_index).map_err(|_| { + CompileError::SetBranchTargetOfNonBranchInstruction { + instruction: format!("{:?}", instruction), + span: *span, + } + })?; + } + } + + Ok(IrBlock { + instructions: self.instructions, + spans: self.spans, + data: self.data.into(), + ast: self.ast, + comments: self.comments.into_iter().map(|s| s.into()).collect(), + register_count: self + .register_allocation_state + .len() + .try_into() + .expect("register count overflowed in finish() despite previous checks"), + file_count: self.file_count, + }) + } +} + +/// Keeps track of the `break` and `continue` target labels for a loop. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Loop { + pub(crate) break_label: LabelId, + pub(crate) continue_label: LabelId, +} + +/// Add a new comment to an existing one +fn add_comment(comment: &mut String, new_comment: impl std::fmt::Display) { + use std::fmt::Write; + write!( + comment, + "{}{}", + if comment.is_empty() { "" } else { ", " }, + new_comment + ) + .expect("formatting failed"); +} diff --git a/crates/nu-engine/src/compile/call.rs b/crates/nu-engine/src/compile/call.rs new file mode 100644 index 0000000000..d9f1b8e581 --- /dev/null +++ b/crates/nu-engine/src/compile/call.rs @@ -0,0 +1,270 @@ +use std::sync::Arc; + +use nu_protocol::{ + ast::{Argument, Call, Expression, ExternalArgument}, + engine::StateWorkingSet, + ir::{Instruction, IrAstRef, Literal}, + IntoSpanned, RegId, Span, Spanned, +}; + +use super::{compile_expression, keyword::*, BlockBuilder, CompileError, RedirectModes}; + +pub(crate) fn compile_call( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + call: &Call, + redirect_modes: RedirectModes, + io_reg: RegId, +) -> Result<(), CompileError> { + let decl = working_set.get_decl(call.decl_id); + + // Check if this call has --help - if so, just redirect to `help` + if call.named_iter().any(|(name, _, _)| name.item == "help") { + return compile_help( + working_set, + builder, + decl.name().into_spanned(call.head), + io_reg, + ); + } + + // Try to figure out if this is a keyword call like `if`, and handle those specially + if decl.is_keyword() { + match decl.name() { + "if" => { + return compile_if(working_set, builder, call, redirect_modes, io_reg); + } + "match" => { + return compile_match(working_set, builder, call, redirect_modes, io_reg); + } + "const" => { + // This differs from the behavior of the const command, which adds the const value + // to the stack. Since `load-variable` also checks `engine_state` for the variable + // and will get a const value though, is it really necessary to do that? + return builder.load_empty(io_reg); + } + "alias" => { + // Alias does nothing + return builder.load_empty(io_reg); + } + "let" | "mut" => { + return compile_let(working_set, builder, call, redirect_modes, io_reg); + } + "try" => { + return compile_try(working_set, builder, call, redirect_modes, io_reg); + } + "loop" => { + return compile_loop(working_set, builder, call, redirect_modes, io_reg); + } + "while" => { + return compile_while(working_set, builder, call, redirect_modes, io_reg); + } + "for" => { + return compile_for(working_set, builder, call, redirect_modes, io_reg); + } + "break" => { + return compile_break(working_set, builder, call, redirect_modes, io_reg); + } + "continue" => { + return compile_continue(working_set, builder, call, redirect_modes, io_reg); + } + "return" => { + return compile_return(working_set, builder, call, redirect_modes, io_reg); + } + _ => (), + } + } + + // Keep AST if the decl needs it. + let requires_ast = decl.requires_ast_for_arguments(); + + // It's important that we evaluate the args first before trying to set up the argument + // state for the call. + // + // We could technically compile anything that isn't another call safely without worrying about + // the argument state, but we'd have to check all of that first and it just isn't really worth + // it. + enum CompiledArg<'a> { + Positional(RegId, Span, Option), + Named( + &'a str, + Option<&'a str>, + Option, + Span, + Option, + ), + Spread(RegId, Span, Option), + } + + let mut compiled_args = vec![]; + + for arg in &call.arguments { + let arg_reg = arg + .expr() + .map(|expr| { + let arg_reg = builder.next_register()?; + + compile_expression( + working_set, + builder, + expr, + RedirectModes::capture_out(arg.span()), + None, + arg_reg, + )?; + + Ok(arg_reg) + }) + .transpose()?; + + let ast_ref = arg + .expr() + .filter(|_| requires_ast) + .map(|expr| IrAstRef(Arc::new(expr.clone()))); + + match arg { + Argument::Positional(_) | Argument::Unknown(_) => { + compiled_args.push(CompiledArg::Positional( + arg_reg.expect("expr() None in non-Named"), + arg.span(), + ast_ref, + )) + } + Argument::Named((name, short, _)) => compiled_args.push(CompiledArg::Named( + &name.item, + short.as_ref().map(|spanned| spanned.item.as_str()), + arg_reg, + arg.span(), + ast_ref, + )), + Argument::Spread(_) => compiled_args.push(CompiledArg::Spread( + arg_reg.expect("expr() None in non-Named"), + arg.span(), + ast_ref, + )), + } + } + + // Now that the args are all compiled, set up the call state (argument stack and redirections) + for arg in compiled_args { + match arg { + CompiledArg::Positional(reg, span, ast_ref) => { + builder.push(Instruction::PushPositional { src: reg }.into_spanned(span))?; + builder.set_last_ast(ast_ref); + } + CompiledArg::Named(name, short, Some(reg), span, ast_ref) => { + if !name.is_empty() { + let name = builder.data(name)?; + builder.push(Instruction::PushNamed { name, src: reg }.into_spanned(span))?; + } else { + let short = builder.data(short.unwrap_or(""))?; + builder + .push(Instruction::PushShortNamed { short, src: reg }.into_spanned(span))?; + } + builder.set_last_ast(ast_ref); + } + CompiledArg::Named(name, short, None, span, ast_ref) => { + if !name.is_empty() { + let name = builder.data(name)?; + builder.push(Instruction::PushFlag { name }.into_spanned(span))?; + } else { + let short = builder.data(short.unwrap_or(""))?; + builder.push(Instruction::PushShortFlag { short }.into_spanned(span))?; + } + builder.set_last_ast(ast_ref); + } + CompiledArg::Spread(reg, span, ast_ref) => { + builder.push(Instruction::AppendRest { src: reg }.into_spanned(span))?; + builder.set_last_ast(ast_ref); + } + } + } + + // Add any parser info from the call + for (name, info) in &call.parser_info { + let name = builder.data(name)?; + let info = Box::new(info.clone()); + builder.push(Instruction::PushParserInfo { name, info }.into_spanned(call.head))?; + } + + if let Some(mode) = redirect_modes.out { + builder.push(mode.map(|mode| Instruction::RedirectOut { mode }))?; + } + + if let Some(mode) = redirect_modes.err { + builder.push(mode.map(|mode| Instruction::RedirectErr { mode }))?; + } + + // The state is set up, so we can do the call into io_reg + builder.push( + Instruction::Call { + decl_id: call.decl_id, + src_dst: io_reg, + } + .into_spanned(call.head), + )?; + + Ok(()) +} + +pub(crate) fn compile_help( + working_set: &StateWorkingSet<'_>, + builder: &mut BlockBuilder, + decl_name: Spanned<&str>, + io_reg: RegId, +) -> Result<(), CompileError> { + let help_command_id = + working_set + .find_decl(b"help") + .ok_or_else(|| CompileError::MissingRequiredDeclaration { + decl_name: "help".into(), + span: decl_name.span, + })?; + + let name_data = builder.data(decl_name.item)?; + let name_literal = builder.literal(decl_name.map(|_| Literal::String(name_data)))?; + + builder.push(Instruction::PushPositional { src: name_literal }.into_spanned(decl_name.span))?; + + builder.push( + Instruction::Call { + decl_id: help_command_id, + src_dst: io_reg, + } + .into_spanned(decl_name.span), + )?; + + Ok(()) +} + +pub(crate) fn compile_external_call( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + head: &Expression, + args: &[ExternalArgument], + redirect_modes: RedirectModes, + io_reg: RegId, +) -> Result<(), CompileError> { + // Pass everything to run-external + let run_external_id = working_set + .find_decl(b"run-external") + .ok_or(CompileError::RunExternalNotFound { span: head.span })?; + + let mut call = Call::new(head.span); + call.decl_id = run_external_id; + + call.arguments.push(Argument::Positional(head.clone())); + + for arg in args { + match arg { + ExternalArgument::Regular(expr) => { + call.arguments.push(Argument::Positional(expr.clone())); + } + ExternalArgument::Spread(expr) => { + call.arguments.push(Argument::Spread(expr.clone())); + } + } + } + + compile_call(working_set, builder, &call, redirect_modes, io_reg) +} diff --git a/crates/nu-engine/src/compile/expression.rs b/crates/nu-engine/src/compile/expression.rs new file mode 100644 index 0000000000..38ee58ea26 --- /dev/null +++ b/crates/nu-engine/src/compile/expression.rs @@ -0,0 +1,535 @@ +use super::{ + compile_binary_op, compile_block, compile_call, compile_external_call, compile_load_env, + BlockBuilder, CompileError, RedirectModes, +}; + +use nu_protocol::{ + ast::{CellPath, Expr, Expression, ListItem, RecordItem, ValueWithUnit}, + engine::StateWorkingSet, + ir::{DataSlice, Instruction, Literal}, + IntoSpanned, RegId, Span, Value, ENV_VARIABLE_ID, +}; + +pub(crate) fn compile_expression( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + expr: &Expression, + redirect_modes: RedirectModes, + in_reg: Option, + out_reg: RegId, +) -> Result<(), CompileError> { + let drop_input = |builder: &mut BlockBuilder| { + if let Some(in_reg) = in_reg { + if in_reg != out_reg { + builder.drop_reg(in_reg)?; + } + } + Ok(()) + }; + + let lit = |builder: &mut BlockBuilder, literal: Literal| { + drop_input(builder)?; + + builder + .push( + Instruction::LoadLiteral { + dst: out_reg, + lit: literal, + } + .into_spanned(expr.span), + ) + .map(|_| ()) + }; + + let ignore = |builder: &mut BlockBuilder| { + drop_input(builder)?; + builder.load_empty(out_reg) + }; + + let unexpected = |expr_name: &str| CompileError::UnexpectedExpression { + expr_name: expr_name.into(), + span: expr.span, + }; + + let move_in_reg_to_out_reg = |builder: &mut BlockBuilder| { + // Ensure that out_reg contains the input value, because a call only uses one register + if let Some(in_reg) = in_reg { + if in_reg != out_reg { + // Have to move in_reg to out_reg so it can be used + builder.push( + Instruction::Move { + dst: out_reg, + src: in_reg, + } + .into_spanned(expr.span), + )?; + } + } else { + // Will have to initialize out_reg with Empty first + builder.load_empty(out_reg)?; + } + Ok(()) + }; + + match &expr.expr { + Expr::Bool(b) => lit(builder, Literal::Bool(*b)), + Expr::Int(i) => lit(builder, Literal::Int(*i)), + Expr::Float(f) => lit(builder, Literal::Float(*f)), + Expr::Binary(bin) => { + let data_slice = builder.data(bin)?; + lit(builder, Literal::Binary(data_slice)) + } + Expr::Range(range) => { + // Compile the subexpressions of the range + let compile_part = |builder: &mut BlockBuilder, + part_expr: Option<&Expression>| + -> Result { + let reg = builder.next_register()?; + if let Some(part_expr) = part_expr { + compile_expression( + working_set, + builder, + part_expr, + RedirectModes::capture_out(part_expr.span), + None, + reg, + )?; + } else { + builder.load_literal(reg, Literal::Nothing.into_spanned(expr.span))?; + } + Ok(reg) + }; + + drop_input(builder)?; + + let start = compile_part(builder, range.from.as_ref())?; + let step = compile_part(builder, range.next.as_ref())?; + let end = compile_part(builder, range.to.as_ref())?; + + // Assemble the range + builder.load_literal( + out_reg, + Literal::Range { + start, + step, + end, + inclusion: range.operator.inclusion, + } + .into_spanned(expr.span), + ) + } + Expr::Var(var_id) => { + drop_input(builder)?; + builder.push( + Instruction::LoadVariable { + dst: out_reg, + var_id: *var_id, + } + .into_spanned(expr.span), + )?; + Ok(()) + } + Expr::VarDecl(_) => Err(unexpected("VarDecl")), + Expr::Call(call) => { + move_in_reg_to_out_reg(builder)?; + + compile_call(working_set, builder, call, redirect_modes, out_reg) + } + Expr::ExternalCall(head, args) => { + move_in_reg_to_out_reg(builder)?; + + compile_external_call(working_set, builder, head, args, redirect_modes, out_reg) + } + Expr::Operator(_) => Err(unexpected("Operator")), + Expr::RowCondition(block_id) => lit(builder, Literal::RowCondition(*block_id)), + Expr::UnaryNot(subexpr) => { + drop_input(builder)?; + compile_expression( + working_set, + builder, + subexpr, + RedirectModes::capture_out(subexpr.span), + None, + out_reg, + )?; + builder.push(Instruction::Not { src_dst: out_reg }.into_spanned(expr.span))?; + Ok(()) + } + Expr::BinaryOp(lhs, op, rhs) => { + if let Expr::Operator(ref operator) = op.expr { + drop_input(builder)?; + compile_binary_op( + working_set, + builder, + lhs, + operator.clone().into_spanned(op.span), + rhs, + expr.span, + out_reg, + ) + } else { + Err(CompileError::UnsupportedOperatorExpression { span: op.span }) + } + } + Expr::Subexpression(block_id) => { + let block = working_set.get_block(*block_id); + compile_block(working_set, builder, block, redirect_modes, in_reg, out_reg) + } + Expr::Block(block_id) => lit(builder, Literal::Block(*block_id)), + Expr::Closure(block_id) => lit(builder, Literal::Closure(*block_id)), + Expr::MatchBlock(_) => Err(unexpected("MatchBlock")), // only for `match` keyword + Expr::List(items) => { + // Guess capacity based on items (does not consider spread as more than 1) + lit( + builder, + Literal::List { + capacity: items.len(), + }, + )?; + for item in items { + // Compile the expression of the item / spread + let reg = builder.next_register()?; + let expr = match item { + ListItem::Item(expr) | ListItem::Spread(_, expr) => expr, + }; + compile_expression( + working_set, + builder, + expr, + RedirectModes::capture_out(expr.span), + None, + reg, + )?; + + match item { + ListItem::Item(_) => { + // Add each item using list-push + builder.push( + Instruction::ListPush { + src_dst: out_reg, + item: reg, + } + .into_spanned(expr.span), + )?; + } + ListItem::Spread(spread_span, _) => { + // Spread the list using list-spread + builder.push( + Instruction::ListSpread { + src_dst: out_reg, + items: reg, + } + .into_spanned(*spread_span), + )?; + } + } + } + Ok(()) + } + Expr::Table(table) => { + lit( + builder, + Literal::List { + capacity: table.rows.len(), + }, + )?; + + // Evaluate the columns + let column_registers = table + .columns + .iter() + .map(|column| { + let reg = builder.next_register()?; + compile_expression( + working_set, + builder, + column, + RedirectModes::capture_out(column.span), + None, + reg, + )?; + Ok(reg) + }) + .collect::, CompileError>>()?; + + // Build records for each row + for row in table.rows.iter() { + let row_reg = builder.next_register()?; + builder.load_literal( + row_reg, + Literal::Record { + capacity: table.columns.len(), + } + .into_spanned(expr.span), + )?; + for (column_reg, item) in column_registers.iter().zip(row.iter()) { + let column_reg = builder.clone_reg(*column_reg, item.span)?; + let item_reg = builder.next_register()?; + compile_expression( + working_set, + builder, + item, + RedirectModes::capture_out(item.span), + None, + item_reg, + )?; + builder.push( + Instruction::RecordInsert { + src_dst: row_reg, + key: column_reg, + val: item_reg, + } + .into_spanned(item.span), + )?; + } + builder.push( + Instruction::ListPush { + src_dst: out_reg, + item: row_reg, + } + .into_spanned(expr.span), + )?; + } + + // Free the column registers, since they aren't needed anymore + for reg in column_registers { + builder.drop_reg(reg)?; + } + + Ok(()) + } + Expr::Record(items) => { + lit( + builder, + Literal::Record { + capacity: items.len(), + }, + )?; + + for item in items { + match item { + RecordItem::Pair(key, val) => { + // Add each item using record-insert + let key_reg = builder.next_register()?; + let val_reg = builder.next_register()?; + compile_expression( + working_set, + builder, + key, + RedirectModes::capture_out(key.span), + None, + key_reg, + )?; + compile_expression( + working_set, + builder, + val, + RedirectModes::capture_out(val.span), + None, + val_reg, + )?; + builder.push( + Instruction::RecordInsert { + src_dst: out_reg, + key: key_reg, + val: val_reg, + } + .into_spanned(expr.span), + )?; + } + RecordItem::Spread(spread_span, expr) => { + // Spread the expression using record-spread + let reg = builder.next_register()?; + compile_expression( + working_set, + builder, + expr, + RedirectModes::capture_out(expr.span), + None, + reg, + )?; + builder.push( + Instruction::RecordSpread { + src_dst: out_reg, + items: reg, + } + .into_spanned(*spread_span), + )?; + } + } + } + Ok(()) + } + Expr::Keyword(kw) => { + // keyword: just pass through expr, since commands that use it and are not being + // specially handled already are often just positional anyway + compile_expression( + working_set, + builder, + &kw.expr, + redirect_modes, + in_reg, + out_reg, + ) + } + Expr::ValueWithUnit(value_with_unit) => { + lit(builder, literal_from_value_with_unit(value_with_unit)?) + } + Expr::DateTime(dt) => lit(builder, Literal::Date(Box::new(*dt))), + Expr::Filepath(path, no_expand) => { + let val = builder.data(path)?; + lit( + builder, + Literal::Filepath { + val, + no_expand: *no_expand, + }, + ) + } + Expr::Directory(path, no_expand) => { + let val = builder.data(path)?; + lit( + builder, + Literal::Directory { + val, + no_expand: *no_expand, + }, + ) + } + Expr::GlobPattern(path, no_expand) => { + let val = builder.data(path)?; + lit( + builder, + Literal::GlobPattern { + val, + no_expand: *no_expand, + }, + ) + } + Expr::String(s) => { + let data_slice = builder.data(s)?; + lit(builder, Literal::String(data_slice)) + } + Expr::RawString(rs) => { + let data_slice = builder.data(rs)?; + lit(builder, Literal::RawString(data_slice)) + } + Expr::CellPath(path) => lit(builder, Literal::CellPath(Box::new(path.clone()))), + Expr::FullCellPath(full_cell_path) => { + if matches!(full_cell_path.head.expr, Expr::Var(ENV_VARIABLE_ID)) { + compile_load_env(builder, expr.span, &full_cell_path.tail, out_reg) + } else { + compile_expression( + working_set, + builder, + &full_cell_path.head, + RedirectModes::capture_out(expr.span), + in_reg, + out_reg, + )?; + // Only do the follow if this is actually needed + if !full_cell_path.tail.is_empty() { + let cell_path_reg = builder.literal( + Literal::CellPath(Box::new(CellPath { + members: full_cell_path.tail.clone(), + })) + .into_spanned(expr.span), + )?; + builder.push( + Instruction::FollowCellPath { + src_dst: out_reg, + path: cell_path_reg, + } + .into_spanned(expr.span), + )?; + } + Ok(()) + } + } + Expr::ImportPattern(_) => Err(unexpected("ImportPattern")), + Expr::Overlay(_) => Err(unexpected("Overlay")), + Expr::Signature(_) => ignore(builder), // no effect + Expr::StringInterpolation(exprs) | Expr::GlobInterpolation(exprs, _) => { + let mut exprs_iter = exprs.iter().peekable(); + + if exprs_iter + .peek() + .is_some_and(|e| matches!(e.expr, Expr::String(..) | Expr::RawString(..))) + { + // If the first expression is a string or raw string literal, just take it and build + // from that + compile_expression( + working_set, + builder, + exprs_iter.next().expect("peek() was Some"), + RedirectModes::capture_out(expr.span), + None, + out_reg, + )?; + } else { + // Start with an empty string + lit(builder, Literal::String(DataSlice::empty()))?; + } + + // Compile each expression and append to out_reg + for expr in exprs_iter { + let scratch_reg = builder.next_register()?; + compile_expression( + working_set, + builder, + expr, + RedirectModes::capture_out(expr.span), + None, + scratch_reg, + )?; + builder.push( + Instruction::StringAppend { + src_dst: out_reg, + val: scratch_reg, + } + .into_spanned(expr.span), + )?; + } + + // If it's a glob interpolation, change it to a glob + if let Expr::GlobInterpolation(_, no_expand) = expr.expr { + builder.push( + Instruction::GlobFrom { + src_dst: out_reg, + no_expand, + } + .into_spanned(expr.span), + )?; + } + + Ok(()) + } + Expr::Nothing => lit(builder, Literal::Nothing), + Expr::Garbage => Err(CompileError::Garbage { span: expr.span }), + } +} + +fn literal_from_value_with_unit(value_with_unit: &ValueWithUnit) -> Result { + let Expr::Int(int_value) = value_with_unit.expr.expr else { + return Err(CompileError::UnexpectedExpression { + expr_name: format!("{:?}", value_with_unit.expr), + span: value_with_unit.expr.span, + }); + }; + + match value_with_unit + .unit + .item + .build_value(int_value, Span::unknown()) + .map_err(|err| CompileError::InvalidLiteral { + msg: err.to_string(), + span: value_with_unit.expr.span, + })? { + Value::Filesize { val, .. } => Ok(Literal::Filesize(val)), + Value::Duration { val, .. } => Ok(Literal::Duration(val)), + other => Err(CompileError::InvalidLiteral { + msg: format!("bad value returned by Unit::build_value(): {other:?}"), + span: value_with_unit.unit.span, + }), + } +} diff --git a/crates/nu-engine/src/compile/keyword.rs b/crates/nu-engine/src/compile/keyword.rs new file mode 100644 index 0000000000..12f4a54c10 --- /dev/null +++ b/crates/nu-engine/src/compile/keyword.rs @@ -0,0 +1,902 @@ +use nu_protocol::{ + ast::{Block, Call, Expr, Expression}, + engine::StateWorkingSet, + ir::Instruction, + IntoSpanned, RegId, Type, VarId, +}; + +use super::{compile_block, compile_expression, BlockBuilder, CompileError, RedirectModes}; + +/// Compile a call to `if` as a branch-if +pub(crate) fn compile_if( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + call: &Call, + redirect_modes: RedirectModes, + io_reg: RegId, +) -> Result<(), CompileError> { + // Pseudocode: + // + // %io_reg <- + // not %io_reg + // branch-if %io_reg, FALSE + // TRUE: ...... + // jump END + // FALSE: ...... OR drop %io_reg + // END: + let invalid = || CompileError::InvalidKeywordCall { + keyword: "if".into(), + span: call.head, + }; + + let condition = call.positional_nth(0).ok_or_else(invalid)?; + let true_block_arg = call.positional_nth(1).ok_or_else(invalid)?; + let else_arg = call.positional_nth(2); + + let true_block_id = true_block_arg.as_block().ok_or_else(invalid)?; + let true_block = working_set.get_block(true_block_id); + + let true_label = builder.label(None); + let false_label = builder.label(None); + let end_label = builder.label(None); + + let not_condition_reg = { + // Compile the condition first + let condition_reg = builder.next_register()?; + compile_expression( + working_set, + builder, + condition, + RedirectModes::capture_out(condition.span), + None, + condition_reg, + )?; + + // Negate the condition - we basically only want to jump if the condition is false + builder.push( + Instruction::Not { + src_dst: condition_reg, + } + .into_spanned(call.head), + )?; + + condition_reg + }; + + // Set up a branch if the condition is false. + builder.branch_if(not_condition_reg, false_label, call.head)?; + builder.add_comment("if false"); + + // Compile the true case + builder.set_label(true_label, builder.here())?; + compile_block( + working_set, + builder, + true_block, + redirect_modes.clone(), + Some(io_reg), + io_reg, + )?; + + // Add a jump over the false case + builder.jump(end_label, else_arg.map(|e| e.span).unwrap_or(call.head))?; + builder.add_comment("end if"); + + // On the else side now, assert that io_reg is still valid + builder.set_label(false_label, builder.here())?; + builder.mark_register(io_reg)?; + + if let Some(else_arg) = else_arg { + let Expression { + expr: Expr::Keyword(else_keyword), + .. + } = else_arg + else { + return Err(invalid()); + }; + + if else_keyword.keyword.as_ref() != b"else" { + return Err(invalid()); + } + + let else_expr = &else_keyword.expr; + + match &else_expr.expr { + Expr::Block(block_id) => { + let false_block = working_set.get_block(*block_id); + compile_block( + working_set, + builder, + false_block, + redirect_modes, + Some(io_reg), + io_reg, + )?; + } + _ => { + // The else case supports bare expressions too, not only blocks + compile_expression( + working_set, + builder, + else_expr, + redirect_modes, + Some(io_reg), + io_reg, + )?; + } + } + } else { + // We don't have an else expression/block, so just set io_reg = Empty + builder.load_empty(io_reg)?; + } + + // Set the end label + builder.set_label(end_label, builder.here())?; + + Ok(()) +} + +/// Compile a call to `match` +pub(crate) fn compile_match( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + call: &Call, + redirect_modes: RedirectModes, + io_reg: RegId, +) -> Result<(), CompileError> { + // Pseudocode: + // + // %match_reg <- + // collect %match_reg + // match (pat1), %match_reg, PAT1 + // MATCH2: match (pat2), %match_reg, PAT2 + // FAIL: drop %io_reg + // drop %match_reg + // jump END + // PAT1: %guard_reg <- + // check-match-guard %guard_reg + // not %guard_reg + // branch-if %guard_reg, MATCH2 + // drop %match_reg + // <...expr...> + // jump END + // PAT2: drop %match_reg + // <...expr...> + // jump END + // END: + let invalid = || CompileError::InvalidKeywordCall { + keyword: "match".into(), + span: call.head, + }; + + let match_expr = call.positional_nth(0).ok_or_else(invalid)?; + + let match_block_arg = call.positional_nth(1).ok_or_else(invalid)?; + let match_block = match_block_arg.as_match_block().ok_or_else(invalid)?; + + let match_reg = builder.next_register()?; + + // Evaluate the match expression (patterns will be checked against this). + compile_expression( + working_set, + builder, + match_expr, + RedirectModes::capture_out(match_expr.span), + None, + match_reg, + )?; + + // Important to collect it first + builder.push(Instruction::Collect { src_dst: match_reg }.into_spanned(match_expr.span))?; + + // Generate the `match` instructions. Guards are not used at this stage. + let mut match_labels = Vec::with_capacity(match_block.len()); + let mut next_labels = Vec::with_capacity(match_block.len()); + let end_label = builder.label(None); + + for (pattern, _) in match_block { + let match_label = builder.label(None); + match_labels.push(match_label); + builder.push( + Instruction::Match { + pattern: Box::new(pattern.pattern.clone()), + src: match_reg, + index: match_label.0, + } + .into_spanned(pattern.span), + )?; + // Also add a label for the next match instruction or failure case + next_labels.push(builder.label(Some(builder.here()))); + } + + // Match fall-through to jump to the end, if no match + builder.load_empty(io_reg)?; + builder.drop_reg(match_reg)?; + builder.jump(end_label, call.head)?; + + // Generate each of the match expressions. Handle guards here, if present. + for (index, (pattern, expr)) in match_block.iter().enumerate() { + let match_label = match_labels[index]; + let next_label = next_labels[index]; + + // `io_reg` and `match_reg` are still valid at each of these branch targets + builder.mark_register(io_reg)?; + builder.mark_register(match_reg)?; + + // Set the original match instruction target here + builder.set_label(match_label, builder.here())?; + + // Handle guard, if present + if let Some(guard) = &pattern.guard { + let guard_reg = builder.next_register()?; + compile_expression( + working_set, + builder, + guard, + RedirectModes::capture_out(guard.span), + None, + guard_reg, + )?; + builder + .push(Instruction::CheckMatchGuard { src: guard_reg }.into_spanned(guard.span))?; + builder.push(Instruction::Not { src_dst: guard_reg }.into_spanned(guard.span))?; + // Branch to the next match instruction if the branch fails to match + builder.branch_if( + guard_reg, + next_label, + // Span the branch with the next pattern, or the head if this is the end + match_block + .get(index + 1) + .map(|b| b.0.span) + .unwrap_or(call.head), + )?; + builder.add_comment("if match guard false"); + } + + // match_reg no longer needed, successful match + builder.drop_reg(match_reg)?; + + // Execute match right hand side expression + if let Some(block_id) = expr.as_block() { + let block = working_set.get_block(block_id); + compile_block( + working_set, + builder, + block, + redirect_modes.clone(), + Some(io_reg), + io_reg, + )?; + } else { + compile_expression( + working_set, + builder, + expr, + redirect_modes.clone(), + Some(io_reg), + io_reg, + )?; + } + + // Jump to the end after the match logic is done + builder.jump(end_label, call.head)?; + builder.add_comment("end match"); + } + + // Set the end destination + builder.set_label(end_label, builder.here())?; + + Ok(()) +} + +/// Compile a call to `let` or `mut` (just do store-variable) +pub(crate) fn compile_let( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + call: &Call, + _redirect_modes: RedirectModes, + io_reg: RegId, +) -> Result<(), CompileError> { + // Pseudocode: + // + // %io_reg <- ...... <- %io_reg + // store-variable $var, %io_reg + let invalid = || CompileError::InvalidKeywordCall { + keyword: "let".into(), + span: call.head, + }; + + let var_decl_arg = call.positional_nth(0).ok_or_else(invalid)?; + let block_arg = call.positional_nth(1).ok_or_else(invalid)?; + + let var_id = var_decl_arg.as_var().ok_or_else(invalid)?; + let block_id = block_arg.as_block().ok_or_else(invalid)?; + let block = working_set.get_block(block_id); + + let variable = working_set.get_variable(var_id); + + compile_block( + working_set, + builder, + block, + RedirectModes::capture_out(call.head), + Some(io_reg), + io_reg, + )?; + + // If the variable is a glob type variable, we should cast it with GlobFrom + if variable.ty == Type::Glob { + builder.push( + Instruction::GlobFrom { + src_dst: io_reg, + no_expand: true, + } + .into_spanned(call.head), + )?; + } + + builder.push( + Instruction::StoreVariable { + var_id, + src: io_reg, + } + .into_spanned(call.head), + )?; + builder.add_comment("let"); + + // Don't forget to set io_reg to Empty afterward, as that's the result of an assignment + builder.load_empty(io_reg)?; + + Ok(()) +} + +/// Compile a call to `try`, setting an error handler over the evaluated block +pub(crate) fn compile_try( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + call: &Call, + redirect_modes: RedirectModes, + io_reg: RegId, +) -> Result<(), CompileError> { + // Pseudocode (literal block): + // + // on-error-into ERR, %io_reg // or without + // %io_reg <- <...block...> <- %io_reg + // check-external-failed %failed_reg, %io_reg + // branch-if %failed_reg, FAIL + // pop-error-handler + // jump END + // FAIL: drain %io_reg + // unreachable + // ERR: clone %err_reg, %io_reg + // store-variable $err_var, %err_reg // or without + // %io_reg <- <...catch block...> <- %io_reg // set to empty if no catch block + // END: + // + // with expression that can't be inlined: + // + // %closure_reg <- + // on-error-into ERR, %io_reg + // %io_reg <- <...block...> <- %io_reg + // check-external-failed %failed_reg, %io_reg + // branch-if %failed_reg, FAIL + // pop-error-handler + // jump END + // FAIL: drain %io_reg + // unreachable + // ERR: clone %err_reg, %io_reg + // push-positional %closure_reg + // push-positional %err_reg + // call "do", %io_reg + // END: + let invalid = || CompileError::InvalidKeywordCall { + keyword: "try".into(), + span: call.head, + }; + + let block_arg = call.positional_nth(0).ok_or_else(invalid)?; + let block_id = block_arg.as_block().ok_or_else(invalid)?; + let block = working_set.get_block(block_id); + + let catch_expr = match call.positional_nth(1) { + Some(kw_expr) => Some(kw_expr.as_keyword().ok_or_else(invalid)?), + None => None, + }; + let catch_span = catch_expr.map(|e| e.span).unwrap_or(call.head); + + let err_label = builder.label(None); + let failed_label = builder.label(None); + let end_label = builder.label(None); + + // We have two ways of executing `catch`: if it was provided as a literal, we can inline it. + // Otherwise, we have to evaluate the expression and keep it as a register, and then call `do`. + enum CatchType<'a> { + Block { + block: &'a Block, + var_id: Option, + }, + Closure { + closure_reg: RegId, + }, + } + + let catch_type = catch_expr + .map(|catch_expr| match catch_expr.as_block() { + Some(block_id) => { + let block = working_set.get_block(block_id); + let var_id = block.signature.get_positional(0).and_then(|v| v.var_id); + Ok(CatchType::Block { block, var_id }) + } + None => { + // We have to compile the catch_expr and use it as a closure + let closure_reg = builder.next_register()?; + compile_expression( + working_set, + builder, + catch_expr, + RedirectModes::capture_out(catch_expr.span), + None, + closure_reg, + )?; + Ok(CatchType::Closure { closure_reg }) + } + }) + .transpose()?; + + // Put the error handler instruction. If we have a catch expression then we should capture the + // error. + if catch_type.is_some() { + builder.push( + Instruction::OnErrorInto { + index: err_label.0, + dst: io_reg, + } + .into_spanned(call.head), + )? + } else { + // Otherwise, we don't need the error value. + builder.push(Instruction::OnError { index: err_label.0 }.into_spanned(call.head))? + }; + + builder.add_comment("try"); + + // Compile the block + compile_block( + working_set, + builder, + block, + redirect_modes.clone(), + Some(io_reg), + io_reg, + )?; + + // Check for external command exit code failure, and also redirect that to the catch handler + let failed_reg = builder.next_register()?; + builder.push( + Instruction::CheckExternalFailed { + dst: failed_reg, + src: io_reg, + } + .into_spanned(catch_span), + )?; + builder.branch_if(failed_reg, failed_label, catch_span)?; + + // Successful case: pop the error handler + builder.push(Instruction::PopErrorHandler.into_spanned(call.head))?; + + // Jump over the failure case + builder.jump(end_label, catch_span)?; + + // Set up an error handler preamble for failed external. + // Draining the %io_reg results in the error handler being called with Empty, and sets + // $env.LAST_EXIT_CODE + builder.set_label(failed_label, builder.here())?; + builder.drain(io_reg, catch_span)?; + builder.add_comment("branches to err"); + builder.unreachable(catch_span)?; + + // This is the real error handler + builder.set_label(err_label, builder.here())?; + + // Mark out register as likely not clean - state in error handler is not well defined + builder.mark_register(io_reg)?; + + // Now compile whatever is necessary for the error handler + match catch_type { + Some(CatchType::Block { block, var_id }) => { + // Error will be in io_reg + builder.mark_register(io_reg)?; + if let Some(var_id) = var_id { + // Take a copy of the error as $err, since it will also be input + let err_reg = builder.next_register()?; + builder.push( + Instruction::Clone { + dst: err_reg, + src: io_reg, + } + .into_spanned(catch_span), + )?; + builder.push( + Instruction::StoreVariable { + var_id, + src: err_reg, + } + .into_spanned(catch_span), + )?; + } + // Compile the block, now that the variable is set + compile_block( + working_set, + builder, + block, + redirect_modes, + Some(io_reg), + io_reg, + )?; + } + Some(CatchType::Closure { closure_reg }) => { + // We should call `do`. Error will be in io_reg + let do_decl_id = working_set.find_decl(b"do").ok_or_else(|| { + CompileError::MissingRequiredDeclaration { + decl_name: "do".into(), + span: call.head, + } + })?; + + // Take a copy of io_reg, because we pass it both as an argument and input + builder.mark_register(io_reg)?; + let err_reg = builder.next_register()?; + builder.push( + Instruction::Clone { + dst: err_reg, + src: io_reg, + } + .into_spanned(catch_span), + )?; + + // Push the closure and the error + builder + .push(Instruction::PushPositional { src: closure_reg }.into_spanned(catch_span))?; + builder.push(Instruction::PushPositional { src: err_reg }.into_spanned(catch_span))?; + + // Call `$err | do $closure $err` + builder.push( + Instruction::Call { + decl_id: do_decl_id, + src_dst: io_reg, + } + .into_spanned(catch_span), + )?; + } + None => { + // Just set out to empty. + builder.load_empty(io_reg)?; + } + } + + // This is the end - if we succeeded, should jump here + builder.set_label(end_label, builder.here())?; + + Ok(()) +} + +/// Compile a call to `loop` (via `jump`) +pub(crate) fn compile_loop( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + call: &Call, + _redirect_modes: RedirectModes, + io_reg: RegId, +) -> Result<(), CompileError> { + // Pseudocode: + // + // drop %io_reg + // LOOP: %io_reg <- ...... + // drain %io_reg + // jump %LOOP + // END: drop %io_reg + let invalid = || CompileError::InvalidKeywordCall { + keyword: "loop".into(), + span: call.head, + }; + + let block_arg = call.positional_nth(0).ok_or_else(invalid)?; + let block_id = block_arg.as_block().ok_or_else(invalid)?; + let block = working_set.get_block(block_id); + + let loop_ = builder.begin_loop(); + builder.load_empty(io_reg)?; + + builder.set_label(loop_.continue_label, builder.here())?; + + compile_block( + working_set, + builder, + block, + RedirectModes::default(), + None, + io_reg, + )?; + + // Drain the output, just like for a semicolon + builder.drain(io_reg, call.head)?; + + builder.jump(loop_.continue_label, call.head)?; + builder.add_comment("loop"); + + builder.set_label(loop_.break_label, builder.here())?; + builder.end_loop(loop_)?; + + // State of %io_reg is not necessarily well defined here due to control flow, so make sure it's + // empty. + builder.mark_register(io_reg)?; + builder.load_empty(io_reg)?; + + Ok(()) +} + +/// Compile a call to `while`, via branch instructions +pub(crate) fn compile_while( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + call: &Call, + _redirect_modes: RedirectModes, + io_reg: RegId, +) -> Result<(), CompileError> { + // Pseudocode: + // + // LOOP: %io_reg <- + // branch-if %io_reg, TRUE + // jump FALSE + // TRUE: %io_reg <- ...... + // drain %io_reg + // jump LOOP + // FALSE: drop %io_reg + let invalid = || CompileError::InvalidKeywordCall { + keyword: "while".into(), + span: call.head, + }; + + let cond_arg = call.positional_nth(0).ok_or_else(invalid)?; + let block_arg = call.positional_nth(1).ok_or_else(invalid)?; + let block_id = block_arg.as_block().ok_or_else(invalid)?; + let block = working_set.get_block(block_id); + + let loop_ = builder.begin_loop(); + builder.set_label(loop_.continue_label, builder.here())?; + + let true_label = builder.label(None); + + compile_expression( + working_set, + builder, + cond_arg, + RedirectModes::capture_out(call.head), + None, + io_reg, + )?; + + builder.branch_if(io_reg, true_label, call.head)?; + builder.add_comment("while"); + builder.jump(loop_.break_label, call.head)?; + builder.add_comment("end while"); + + builder.set_label(true_label, builder.here())?; + + compile_block( + working_set, + builder, + block, + RedirectModes::default(), + None, + io_reg, + )?; + + // Drain the result, just like for a semicolon + builder.drain(io_reg, call.head)?; + + builder.jump(loop_.continue_label, call.head)?; + builder.add_comment("while"); + + builder.set_label(loop_.break_label, builder.here())?; + builder.end_loop(loop_)?; + + // State of %io_reg is not necessarily well defined here due to control flow, so make sure it's + // empty. + builder.mark_register(io_reg)?; + builder.load_empty(io_reg)?; + + Ok(()) +} + +/// Compile a call to `for` (via `iterate`) +pub(crate) fn compile_for( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + call: &Call, + _redirect_modes: RedirectModes, + io_reg: RegId, +) -> Result<(), CompileError> { + // Pseudocode: + // + // %stream_reg <- + // LOOP: iterate %io_reg, %stream_reg, END + // store-variable $var, %io_reg + // %io_reg <- <...block...> + // drain %io_reg + // jump LOOP + // END: drop %io_reg + let invalid = || CompileError::InvalidKeywordCall { + keyword: "for".into(), + span: call.head, + }; + + if call.get_named_arg("numbered").is_some() { + // This is deprecated and we don't support it. + return Err(invalid()); + } + + let var_decl_arg = call.positional_nth(0).ok_or_else(invalid)?; + let var_id = var_decl_arg.as_var().ok_or_else(invalid)?; + + let in_arg = call.positional_nth(1).ok_or_else(invalid)?; + let in_expr = in_arg.as_keyword().ok_or_else(invalid)?; + + let block_arg = call.positional_nth(2).ok_or_else(invalid)?; + let block_id = block_arg.as_block().ok_or_else(invalid)?; + let block = working_set.get_block(block_id); + + // Ensure io_reg is marked so we don't use it + builder.mark_register(io_reg)?; + + let stream_reg = builder.next_register()?; + + compile_expression( + working_set, + builder, + in_expr, + RedirectModes::capture_out(in_expr.span), + None, + stream_reg, + )?; + + // Set up loop state + let loop_ = builder.begin_loop(); + builder.set_label(loop_.continue_label, builder.here())?; + + // This gets a value from the stream each time it's executed + // io_reg basically will act as our scratch register here + builder.push( + Instruction::Iterate { + dst: io_reg, + stream: stream_reg, + end_index: loop_.break_label.0, + } + .into_spanned(call.head), + )?; + builder.add_comment("for"); + + // Put the received value in the variable + builder.push( + Instruction::StoreVariable { + var_id, + src: io_reg, + } + .into_spanned(var_decl_arg.span), + )?; + + // Do the body of the block + compile_block( + working_set, + builder, + block, + RedirectModes::default(), + None, + io_reg, + )?; + + // Drain the output, just like for a semicolon + builder.drain(io_reg, call.head)?; + + // Loop back to iterate to get the next value + builder.jump(loop_.continue_label, call.head)?; + + // Set the end of the loop + builder.set_label(loop_.break_label, builder.here())?; + builder.end_loop(loop_)?; + + // We don't need stream_reg anymore, after the loop + // io_reg may or may not be empty, so be sure it is + builder.free_register(stream_reg)?; + builder.mark_register(io_reg)?; + builder.load_empty(io_reg)?; + + Ok(()) +} + +/// Compile a call to `break`. +pub(crate) fn compile_break( + _working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + call: &Call, + _redirect_modes: RedirectModes, + io_reg: RegId, +) -> Result<(), CompileError> { + if builder.is_in_loop() { + builder.load_empty(io_reg)?; + builder.push_break(call.head)?; + builder.add_comment("break"); + } else { + // Fall back to calling the command if we can't find the loop target statically + builder.push( + Instruction::Call { + decl_id: call.decl_id, + src_dst: io_reg, + } + .into_spanned(call.head), + )?; + } + Ok(()) +} + +/// Compile a call to `continue`. +pub(crate) fn compile_continue( + _working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + call: &Call, + _redirect_modes: RedirectModes, + io_reg: RegId, +) -> Result<(), CompileError> { + if builder.is_in_loop() { + builder.load_empty(io_reg)?; + builder.push_continue(call.head)?; + builder.add_comment("continue"); + } else { + // Fall back to calling the command if we can't find the loop target statically + builder.push( + Instruction::Call { + decl_id: call.decl_id, + src_dst: io_reg, + } + .into_spanned(call.head), + )?; + } + Ok(()) +} + +/// Compile a call to `return` as a `return-early` instruction. +/// +/// This is not strictly necessary, but it is more efficient. +pub(crate) fn compile_return( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + call: &Call, + _redirect_modes: RedirectModes, + io_reg: RegId, +) -> Result<(), CompileError> { + // Pseudocode: + // + // %io_reg <- + // return-early %io_reg + if let Some(arg_expr) = call.positional_nth(0) { + compile_expression( + working_set, + builder, + arg_expr, + RedirectModes::capture_out(arg_expr.span), + None, + io_reg, + )?; + } else { + builder.load_empty(io_reg)?; + } + + // TODO: It would be nice if this could be `return` instead, but there is a little bit of + // behaviour remaining that still depends on `ShellError::Return` + builder.push(Instruction::ReturnEarly { src: io_reg }.into_spanned(call.head))?; + + // io_reg is supposed to remain allocated + builder.load_empty(io_reg)?; + + Ok(()) +} diff --git a/crates/nu-engine/src/compile/mod.rs b/crates/nu-engine/src/compile/mod.rs new file mode 100644 index 0000000000..8f6ae22682 --- /dev/null +++ b/crates/nu-engine/src/compile/mod.rs @@ -0,0 +1,204 @@ +use nu_protocol::{ + ast::{Block, Pipeline, PipelineRedirection, RedirectionSource, RedirectionTarget}, + engine::StateWorkingSet, + ir::{Instruction, IrBlock, RedirectMode}, + CompileError, IntoSpanned, RegId, Span, +}; + +mod builder; +mod call; +mod expression; +mod keyword; +mod operator; +mod redirect; + +use builder::BlockBuilder; +use call::*; +use expression::compile_expression; +use operator::*; +use redirect::*; + +const BLOCK_INPUT: RegId = RegId(0); + +/// Compile Nushell pipeline abstract syntax tree (AST) to internal representation (IR) instructions +/// for evaluation. +pub fn compile(working_set: &StateWorkingSet, block: &Block) -> Result { + let mut builder = BlockBuilder::new(block.span); + + let span = block.span.unwrap_or(Span::unknown()); + + compile_block( + working_set, + &mut builder, + block, + RedirectModes::caller(span), + Some(BLOCK_INPUT), + BLOCK_INPUT, + )?; + + // A complete block has to end with a `return` + builder.push(Instruction::Return { src: BLOCK_INPUT }.into_spanned(span))?; + + builder.finish() +} + +/// Compiles a [`Block`] in-place into an IR block. This can be used in a nested manner, for example +/// by [`compile_if()`], where the instructions for the blocks for the if/else are inlined into the +/// top-level IR block. +fn compile_block( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + block: &Block, + redirect_modes: RedirectModes, + in_reg: Option, + out_reg: RegId, +) -> Result<(), CompileError> { + let span = block.span.unwrap_or(Span::unknown()); + let mut redirect_modes = Some(redirect_modes); + if !block.pipelines.is_empty() { + let last_index = block.pipelines.len() - 1; + for (index, pipeline) in block.pipelines.iter().enumerate() { + compile_pipeline( + working_set, + builder, + pipeline, + span, + // the redirect mode only applies to the last pipeline. + if index == last_index { + redirect_modes + .take() + .expect("should only take redirect_modes once") + } else { + RedirectModes::default() + }, + // input is only passed to the first pipeline. + if index == 0 { in_reg } else { None }, + out_reg, + )?; + + if index != last_index { + // Explicitly drain the out reg after each non-final pipeline, because that's how + // the semicolon functions. + if builder.is_allocated(out_reg) { + builder.push(Instruction::Drain { src: out_reg }.into_spanned(span))?; + } + builder.load_empty(out_reg)?; + } + } + Ok(()) + } else if in_reg.is_none() { + builder.load_empty(out_reg) + } else { + Ok(()) + } +} + +fn compile_pipeline( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + pipeline: &Pipeline, + fallback_span: Span, + redirect_modes: RedirectModes, + in_reg: Option, + out_reg: RegId, +) -> Result<(), CompileError> { + let mut iter = pipeline.elements.iter().peekable(); + let mut in_reg = in_reg; + let mut redirect_modes = Some(redirect_modes); + while let Some(element) = iter.next() { + let span = element.pipe.unwrap_or(fallback_span); + + // We have to get the redirection mode from either the explicit redirection in the pipeline + // element, or from the next expression if it's specified there. If this is the last + // element, then it's from whatever is passed in as the mode to use. + + let next_redirect_modes = if let Some(next_element) = iter.peek() { + let mut modes = redirect_modes_of_expression(working_set, &next_element.expr, span)?; + + // If there's a next element with no inherent redirection we always pipe out *unless* + // this is a single redirection of stderr to pipe (e>|) + if modes.out.is_none() + && !matches!( + element.redirection, + Some(PipelineRedirection::Single { + source: RedirectionSource::Stderr, + target: RedirectionTarget::Pipe { .. } + }) + ) + { + let pipe_span = next_element.pipe.unwrap_or(next_element.expr.span); + modes.out = Some(RedirectMode::Pipe.into_spanned(pipe_span)); + } + + modes + } else { + redirect_modes + .take() + .expect("should only take redirect_modes once") + }; + + let spec_redirect_modes = match &element.redirection { + Some(PipelineRedirection::Single { source, target }) => { + let mode = redirection_target_to_mode(working_set, builder, target)?; + match source { + RedirectionSource::Stdout => RedirectModes { + out: Some(mode), + err: None, + }, + RedirectionSource::Stderr => RedirectModes { + out: None, + err: Some(mode), + }, + RedirectionSource::StdoutAndStderr => RedirectModes { + out: Some(mode), + err: Some(mode), + }, + } + } + Some(PipelineRedirection::Separate { out, err }) => { + // In this case, out and err must not both be Pipe + assert!( + !matches!( + (out, err), + ( + RedirectionTarget::Pipe { .. }, + RedirectionTarget::Pipe { .. } + ) + ), + "for Separate redirection, out and err targets must not both be Pipe" + ); + let out = redirection_target_to_mode(working_set, builder, out)?; + let err = redirection_target_to_mode(working_set, builder, err)?; + RedirectModes { + out: Some(out), + err: Some(err), + } + } + None => RedirectModes { + out: None, + err: None, + }, + }; + + let redirect_modes = RedirectModes { + out: spec_redirect_modes.out.or(next_redirect_modes.out), + err: spec_redirect_modes.err.or(next_redirect_modes.err), + }; + + compile_expression( + working_set, + builder, + &element.expr, + redirect_modes.clone(), + in_reg, + out_reg, + )?; + + // Clean up the redirection + finish_redirection(builder, redirect_modes, out_reg)?; + + // The next pipeline element takes input from this output + in_reg = Some(out_reg); + } + Ok(()) +} diff --git a/crates/nu-engine/src/compile/operator.rs b/crates/nu-engine/src/compile/operator.rs new file mode 100644 index 0000000000..a1ed3f66df --- /dev/null +++ b/crates/nu-engine/src/compile/operator.rs @@ -0,0 +1,378 @@ +use nu_protocol::{ + ast::{Assignment, Boolean, CellPath, Expr, Expression, Math, Operator, PathMember}, + engine::StateWorkingSet, + ir::{Instruction, Literal}, + IntoSpanned, RegId, Span, Spanned, ENV_VARIABLE_ID, +}; +use nu_utils::IgnoreCaseExt; + +use super::{compile_expression, BlockBuilder, CompileError, RedirectModes}; + +pub(crate) fn compile_binary_op( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + lhs: &Expression, + op: Spanned, + rhs: &Expression, + span: Span, + out_reg: RegId, +) -> Result<(), CompileError> { + if let Operator::Assignment(assign_op) = op.item { + if let Some(decomposed_op) = decompose_assignment(assign_op) { + // Compiling an assignment that uses a binary op with the existing value + compile_binary_op( + working_set, + builder, + lhs, + decomposed_op.into_spanned(op.span), + rhs, + span, + out_reg, + )?; + } else { + // Compiling a plain assignment, where the current left-hand side value doesn't matter + compile_expression( + working_set, + builder, + rhs, + RedirectModes::capture_out(rhs.span), + None, + out_reg, + )?; + } + + compile_assignment(working_set, builder, lhs, op.span, out_reg)?; + + // Load out_reg with Nothing, as that's the result of an assignment + builder.load_literal(out_reg, Literal::Nothing.into_spanned(op.span)) + } else { + // Not an assignment: just do the binary op + let lhs_reg = out_reg; + + compile_expression( + working_set, + builder, + lhs, + RedirectModes::capture_out(lhs.span), + None, + lhs_reg, + )?; + + match op.item { + // `and` / `or` are short-circuiting, and we can get by with one register and a branch + Operator::Boolean(Boolean::And) => { + let true_label = builder.label(None); + builder.branch_if(lhs_reg, true_label, op.span)?; + + // If the branch was not taken it's false, so short circuit to load false + let false_label = builder.label(None); + builder.jump(false_label, op.span)?; + + builder.set_label(true_label, builder.here())?; + compile_expression( + working_set, + builder, + rhs, + RedirectModes::capture_out(rhs.span), + None, + lhs_reg, + )?; + + let end_label = builder.label(None); + builder.jump(end_label, op.span)?; + + // Consumed by `branch-if`, so we have to set it false again + builder.set_label(false_label, builder.here())?; + builder.load_literal(lhs_reg, Literal::Bool(false).into_spanned(lhs.span))?; + + builder.set_label(end_label, builder.here())?; + } + Operator::Boolean(Boolean::Or) => { + let true_label = builder.label(None); + builder.branch_if(lhs_reg, true_label, op.span)?; + + // If the branch was not taken it's false, so do the right-side expression + compile_expression( + working_set, + builder, + rhs, + RedirectModes::capture_out(rhs.span), + None, + lhs_reg, + )?; + + let end_label = builder.label(None); + builder.jump(end_label, op.span)?; + + // Consumed by `branch-if`, so we have to set it true again + builder.set_label(true_label, builder.here())?; + builder.load_literal(lhs_reg, Literal::Bool(true).into_spanned(lhs.span))?; + + builder.set_label(end_label, builder.here())?; + } + _ => { + // Any other operator, via `binary-op` + let rhs_reg = builder.next_register()?; + + compile_expression( + working_set, + builder, + rhs, + RedirectModes::capture_out(rhs.span), + None, + rhs_reg, + )?; + + builder.push( + Instruction::BinaryOp { + lhs_dst: lhs_reg, + op: op.item, + rhs: rhs_reg, + } + .into_spanned(op.span), + )?; + } + } + + if lhs_reg != out_reg { + builder.push( + Instruction::Move { + dst: out_reg, + src: lhs_reg, + } + .into_spanned(op.span), + )?; + } + + builder.push(Instruction::Span { src_dst: out_reg }.into_spanned(span))?; + + Ok(()) + } +} + +/// The equivalent plain operator to use for an assignment, if any +pub(crate) fn decompose_assignment(assignment: Assignment) -> Option { + match assignment { + Assignment::Assign => None, + Assignment::PlusAssign => Some(Operator::Math(Math::Plus)), + Assignment::AppendAssign => Some(Operator::Math(Math::Append)), + Assignment::MinusAssign => Some(Operator::Math(Math::Minus)), + Assignment::MultiplyAssign => Some(Operator::Math(Math::Multiply)), + Assignment::DivideAssign => Some(Operator::Math(Math::Divide)), + } +} + +/// Compile assignment of the value in a register to a left-hand expression +pub(crate) fn compile_assignment( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + lhs: &Expression, + assignment_span: Span, + rhs_reg: RegId, +) -> Result<(), CompileError> { + match lhs.expr { + Expr::Var(var_id) => { + // Double check that the variable is supposed to be mutable + if !working_set.get_variable(var_id).mutable { + return Err(CompileError::AssignmentRequiresMutableVar { span: lhs.span }); + } + + builder.push( + Instruction::StoreVariable { + var_id, + src: rhs_reg, + } + .into_spanned(assignment_span), + )?; + Ok(()) + } + Expr::FullCellPath(ref path) => match (&path.head, &path.tail) { + ( + Expression { + expr: Expr::Var(var_id), + .. + }, + _, + ) if *var_id == ENV_VARIABLE_ID => { + // This will be an assignment to an environment variable. + let Some(PathMember::String { val: key, .. }) = path.tail.first() else { + return Err(CompileError::CannotReplaceEnv { span: lhs.span }); + }; + + // Some env vars can't be set by Nushell code. + const AUTOMATIC_NAMES: &[&str] = &["PWD", "FILE_PWD", "CURRENT_FILE"]; + if AUTOMATIC_NAMES.iter().any(|name| key.eq_ignore_case(name)) { + return Err(CompileError::AutomaticEnvVarSetManually { + envvar_name: "PWD".into(), + span: lhs.span, + }); + } + + let key_data = builder.data(key)?; + + let val_reg = if path.tail.len() > 1 { + // Get the current value of the head and first tail of the path, from env + let head_reg = builder.next_register()?; + + // We could use compile_load_env, but this shares the key data... + // Always use optional, because it doesn't matter if it's already there + builder.push( + Instruction::LoadEnvOpt { + dst: head_reg, + key: key_data, + } + .into_spanned(lhs.span), + )?; + + // Default to empty record so we can do further upserts + let default_label = builder.label(None); + let upsert_label = builder.label(None); + builder.branch_if_empty(head_reg, default_label, assignment_span)?; + builder.jump(upsert_label, assignment_span)?; + + builder.set_label(default_label, builder.here())?; + builder.load_literal( + head_reg, + Literal::Record { capacity: 0 }.into_spanned(lhs.span), + )?; + + // Do the upsert on the current value to incorporate rhs + builder.set_label(upsert_label, builder.here())?; + compile_upsert_cell_path( + builder, + (&path.tail[1..]).into_spanned(lhs.span), + head_reg, + rhs_reg, + assignment_span, + )?; + + head_reg + } else { + // Path has only one tail, so we don't need the current value to do an upsert, + // just set it directly to rhs + rhs_reg + }; + + // Finally, store the modified env variable + builder.push( + Instruction::StoreEnv { + key: key_data, + src: val_reg, + } + .into_spanned(assignment_span), + )?; + Ok(()) + } + (_, tail) if tail.is_empty() => { + // If the path tail is empty, we can really just treat this as if it were an + // assignment to the head + compile_assignment(working_set, builder, &path.head, assignment_span, rhs_reg) + } + _ => { + // Just a normal assignment to some path + let head_reg = builder.next_register()?; + + // Compile getting current value of the head expression + compile_expression( + working_set, + builder, + &path.head, + RedirectModes::capture_out(path.head.span), + None, + head_reg, + )?; + + // Upsert the tail of the path into the old value of the head expression + compile_upsert_cell_path( + builder, + path.tail.as_slice().into_spanned(lhs.span), + head_reg, + rhs_reg, + assignment_span, + )?; + + // Now compile the assignment of the updated value to the head + compile_assignment(working_set, builder, &path.head, assignment_span, head_reg) + } + }, + Expr::Garbage => Err(CompileError::Garbage { span: lhs.span }), + _ => Err(CompileError::AssignmentRequiresVar { span: lhs.span }), + } +} + +/// Compile an upsert-cell-path instruction, with known literal members +pub(crate) fn compile_upsert_cell_path( + builder: &mut BlockBuilder, + members: Spanned<&[PathMember]>, + src_dst: RegId, + new_value: RegId, + span: Span, +) -> Result<(), CompileError> { + let path_reg = builder.literal( + Literal::CellPath( + CellPath { + members: members.item.to_vec(), + } + .into(), + ) + .into_spanned(members.span), + )?; + builder.push( + Instruction::UpsertCellPath { + src_dst, + path: path_reg, + new_value, + } + .into_spanned(span), + )?; + Ok(()) +} + +/// Compile the correct sequence to get an environment variable + follow a path on it +pub(crate) fn compile_load_env( + builder: &mut BlockBuilder, + span: Span, + path: &[PathMember], + out_reg: RegId, +) -> Result<(), CompileError> { + if path.is_empty() { + builder.push( + Instruction::LoadVariable { + dst: out_reg, + var_id: ENV_VARIABLE_ID, + } + .into_spanned(span), + )?; + } else { + let (key, optional) = match &path[0] { + PathMember::String { val, optional, .. } => (builder.data(val)?, *optional), + PathMember::Int { span, .. } => { + return Err(CompileError::AccessEnvByInt { span: *span }) + } + }; + let tail = &path[1..]; + + if optional { + builder.push(Instruction::LoadEnvOpt { dst: out_reg, key }.into_spanned(span))?; + } else { + builder.push(Instruction::LoadEnv { dst: out_reg, key }.into_spanned(span))?; + } + + if !tail.is_empty() { + let path = builder.literal( + Literal::CellPath(Box::new(CellPath { + members: tail.to_vec(), + })) + .into_spanned(span), + )?; + builder.push( + Instruction::FollowCellPath { + src_dst: out_reg, + path, + } + .into_spanned(span), + )?; + } + } + Ok(()) +} diff --git a/crates/nu-engine/src/compile/redirect.rs b/crates/nu-engine/src/compile/redirect.rs new file mode 100644 index 0000000000..15af1a9f8c --- /dev/null +++ b/crates/nu-engine/src/compile/redirect.rs @@ -0,0 +1,157 @@ +use nu_protocol::{ + ast::{Expression, RedirectionTarget}, + engine::StateWorkingSet, + ir::{Instruction, RedirectMode}, + IntoSpanned, OutDest, RegId, Span, Spanned, +}; + +use super::{compile_expression, BlockBuilder, CompileError}; + +#[derive(Default, Clone)] +pub(crate) struct RedirectModes { + pub(crate) out: Option>, + pub(crate) err: Option>, +} + +impl RedirectModes { + pub(crate) fn capture_out(span: Span) -> Self { + RedirectModes { + out: Some(RedirectMode::Capture.into_spanned(span)), + err: None, + } + } + + pub(crate) fn caller(span: Span) -> RedirectModes { + RedirectModes { + out: Some(RedirectMode::Caller.into_spanned(span)), + err: Some(RedirectMode::Caller.into_spanned(span)), + } + } +} + +pub(crate) fn redirection_target_to_mode( + working_set: &StateWorkingSet, + builder: &mut BlockBuilder, + target: &RedirectionTarget, +) -> Result, CompileError> { + Ok(match target { + RedirectionTarget::File { + expr, + append, + span: redir_span, + } => { + let file_num = builder.next_file_num()?; + let path_reg = builder.next_register()?; + compile_expression( + working_set, + builder, + expr, + RedirectModes::capture_out(*redir_span), + None, + path_reg, + )?; + builder.push( + Instruction::OpenFile { + file_num, + path: path_reg, + append: *append, + } + .into_spanned(*redir_span), + )?; + RedirectMode::File { file_num }.into_spanned(*redir_span) + } + RedirectionTarget::Pipe { span } => RedirectMode::Pipe.into_spanned(*span), + }) +} + +pub(crate) fn redirect_modes_of_expression( + working_set: &StateWorkingSet, + expression: &Expression, + redir_span: Span, +) -> Result { + let (out, err) = expression.expr.pipe_redirection(working_set); + Ok(RedirectModes { + out: out + .map(|r| r.into_spanned(redir_span)) + .map(out_dest_to_redirect_mode) + .transpose()?, + err: err + .map(|r| r.into_spanned(redir_span)) + .map(out_dest_to_redirect_mode) + .transpose()?, + }) +} + +/// Finish the redirection for an expression, writing to and closing files as necessary +pub(crate) fn finish_redirection( + builder: &mut BlockBuilder, + modes: RedirectModes, + out_reg: RegId, +) -> Result<(), CompileError> { + if let Some(Spanned { + item: RedirectMode::File { file_num }, + span, + }) = modes.out + { + // If out is a file and err is a pipe, we must not consume the expression result - + // that is actually the err, in that case. + if !matches!( + modes.err, + Some(Spanned { + item: RedirectMode::Pipe { .. }, + .. + }) + ) { + builder.push( + Instruction::WriteFile { + file_num, + src: out_reg, + } + .into_spanned(span), + )?; + builder.load_empty(out_reg)?; + } + builder.push(Instruction::CloseFile { file_num }.into_spanned(span))?; + } + + match modes.err { + Some(Spanned { + item: RedirectMode::File { file_num }, + span, + }) => { + // Close the file, unless it's the same as out (in which case it was already closed) + if !modes.out.is_some_and(|out_mode| match out_mode.item { + RedirectMode::File { + file_num: out_file_num, + } => file_num == out_file_num, + _ => false, + }) { + builder.push(Instruction::CloseFile { file_num }.into_spanned(span))?; + } + } + Some(Spanned { + item: RedirectMode::Pipe, + span, + }) => { + builder.push(Instruction::CheckErrRedirected { src: out_reg }.into_spanned(span))?; + } + _ => (), + } + + Ok(()) +} + +pub(crate) fn out_dest_to_redirect_mode( + out_dest: Spanned, +) -> Result, CompileError> { + let span = out_dest.span; + out_dest + .map(|out_dest| match out_dest { + OutDest::Pipe => Ok(RedirectMode::Pipe), + OutDest::Capture => Ok(RedirectMode::Capture), + OutDest::Null => Ok(RedirectMode::Null), + OutDest::Inherit => Ok(RedirectMode::Inherit), + OutDest::File(_) => Err(CompileError::InvalidRedirectMode { span }), + }) + .transpose() +} diff --git a/crates/nu-engine/src/documentation.rs b/crates/nu-engine/src/documentation.rs index 3cd1130060..7840d03c47 100644 --- a/crates/nu-engine/src/documentation.rs +++ b/crates/nu-engine/src/documentation.rs @@ -2,9 +2,9 @@ use crate::eval_call; use nu_protocol::{ ast::{Argument, Call, Expr, Expression, RecordItem}, debugger::WithoutDebug, - engine::{Command, EngineState, Stack}, - record, Category, Example, IntoPipelineData, PipelineData, Signature, Span, SyntaxShape, Type, - Value, + engine::{Command, EngineState, Stack, UNKNOWN_SPAN_ID}, + record, Category, Example, IntoPipelineData, PipelineData, Signature, Span, SpanId, Spanned, + SyntaxShape, Type, Value, }; use std::{collections::HashMap, fmt::Write}; @@ -45,10 +45,12 @@ fn nu_highlight_string(code_string: &str, engine_state: &EngineState, stack: &mu if let Some(highlighter) = engine_state.find_decl(b"nu-highlight", &[]) { let decl = engine_state.get_decl(highlighter); + let call = Call::new(Span::unknown()); + if let Ok(output) = decl.run( engine_state, stack, - &Call::new(Span::unknown()), + &(&call).into(), Value::string(code_string, Span::unknown()).into_pipeline_data(), ) { let result = output.into_value(Span::unknown()); @@ -269,11 +271,12 @@ fn get_documentation( let _ = write!(long_desc, "\n > {}\n", example.example); } else if let Some(highlighter) = engine_state.find_decl(b"nu-highlight", &[]) { let decl = engine_state.get_decl(highlighter); + let call = Call::new(Span::unknown()); match decl.run( engine_state, stack, - &Call::new(Span::unknown()), + &(&call).into(), Value::string(example.example, Span::unknown()).into_pipeline_data(), ) { Ok(output) => { @@ -296,6 +299,28 @@ fn get_documentation( } if let Some(result) = &example.result { + let mut table_call = Call::new(Span::unknown()); + if example.example.ends_with("--collapse") { + // collapse the result + table_call.add_named(( + Spanned { + item: "collapse".to_string(), + span: Span::unknown(), + }, + None, + None, + )) + } else { + // expand the result + table_call.add_named(( + Spanned { + item: "expand".to_string(), + span: Span::unknown(), + }, + None, + None, + )) + } let table = engine_state .find_decl("table".as_bytes(), &[]) .and_then(|decl_id| { @@ -304,7 +329,7 @@ fn get_documentation( .run( engine_state, stack, - &Call::new(Span::new(0, 0)), + &(&table_call).into(), PipelineData::Value(result.clone(), None), ) .ok() @@ -339,8 +364,9 @@ fn get_ansi_color_for_component_or_default( if let Some(color) = &engine_state.get_config().color_config.get(theme_component) { let caller_stack = &mut Stack::new().capture(); let span = Span::unknown(); + let span_id = UNKNOWN_SPAN_ID; - let argument_opt = get_argument_for_color_value(engine_state, color, span); + let argument_opt = get_argument_for_color_value(engine_state, color, span, span_id); // Call ansi command using argument if let Some(argument) = argument_opt { @@ -371,6 +397,7 @@ fn get_argument_for_color_value( engine_state: &EngineState, color: &&Value, span: Span, + span_id: SpanId, ) -> Option { match color { Value::Record { val, .. } => { @@ -378,43 +405,43 @@ fn get_argument_for_color_value( .iter() .map(|(k, v)| { RecordItem::Pair( - Expression { - expr: Expr::String(k.clone()), + Expression::new_existing( + Expr::String(k.clone()), span, - ty: Type::String, - custom_completion: None, - }, - Expression { - expr: Expr::String( + span_id, + Type::String, + ), + Expression::new_existing( + Expr::String( v.clone().to_expanded_string("", engine_state.get_config()), ), span, - ty: Type::String, - custom_completion: None, - }, + span_id, + Type::String, + ), ) }) .collect(); - Some(Argument::Positional(Expression { - span: Span::unknown(), - ty: Type::Record( + Some(Argument::Positional(Expression::new_existing( + Expr::Record(record_exp), + Span::unknown(), + UNKNOWN_SPAN_ID, + Type::Record( [ ("fg".to_string(), Type::String), ("attr".to_string(), Type::String), ] .into(), ), - expr: Expr::Record(record_exp), - custom_completion: None, - })) + ))) } - Value::String { val, .. } => Some(Argument::Positional(Expression { - span: Span::unknown(), - ty: Type::String, - expr: Expr::String(val.clone()), - custom_completion: None, - })), + Value::String { val, .. } => Some(Argument::Positional(Expression::new_existing( + Expr::String(val.clone()), + Span::unknown(), + UNKNOWN_SPAN_ID, + Type::String, + ))), _ => None, } } diff --git a/crates/nu-engine/src/env.rs b/crates/nu-engine/src/env.rs index 048d9bfb99..ab3a4bc50c 100644 --- a/crates/nu-engine/src/env.rs +++ b/crates/nu-engine/src/env.rs @@ -1,8 +1,8 @@ use crate::ClosureEvalOnce; use nu_path::canonicalize_with; use nu_protocol::{ - ast::{Call, Expr}, - engine::{EngineState, Stack, StateWorkingSet}, + ast::Expr, + engine::{Call, EngineState, Stack, StateWorkingSet}, Config, ShellError, Span, Value, VarId, }; use std::{ @@ -244,14 +244,15 @@ pub fn path_str( } pub const DIR_VAR_PARSER_INFO: &str = "dirs_var"; -pub fn get_dirs_var_from_call(call: &Call) -> Option { - call.get_parser_info(DIR_VAR_PARSER_INFO).and_then(|x| { - if let Expr::Var(id) = x.expr { - Some(id) - } else { - None - } - }) +pub fn get_dirs_var_from_call(stack: &Stack, call: &Call) -> Option { + call.get_parser_info(stack, DIR_VAR_PARSER_INFO) + .and_then(|x| { + if let Expr::Var(id) = x.expr { + Some(id) + } else { + None + } + }) } /// This helper function is used to find files during eval diff --git a/crates/nu-engine/src/eval.rs b/crates/nu-engine/src/eval.rs index af051a1dc7..6e171eb46c 100644 --- a/crates/nu-engine/src/eval.rs +++ b/crates/nu-engine/src/eval.rs @@ -1,13 +1,14 @@ +use crate::eval_ir_block; #[allow(deprecated)] use crate::{current_dir, get_config, get_full_help}; -use nu_path::expand_path_with; +use nu_path::{expand_path_with, AbsolutePathBuf}; use nu_protocol::{ ast::{ Assignment, Block, Call, Expr, Expression, ExternalArgument, PathMember, PipelineElement, PipelineRedirection, RedirectionSource, RedirectionTarget, }, debugger::DebugContext, - engine::{Closure, EngineState, Redirection, Stack}, + engine::{Closure, EngineState, Redirection, Stack, StateWorkingSet}, eval_base::Eval, ByteStreamSource, Config, FromValue, IntoPipelineData, OutDest, PipelineData, ShellError, Span, Spanned, Type, Value, VarId, ENV_VARIABLE_ID, @@ -21,9 +22,7 @@ pub fn eval_call( call: &Call, input: PipelineData, ) -> Result { - if nu_utils::ctrl_c::was_pressed(&engine_state.ctrlc) { - return Ok(Value::nothing(call.head).into_pipeline_data()); - } + engine_state.signals().check(call.head)?; let decl = engine_state.get_decl(call.decl_id); if !decl.is_known_external() && call.named_iter().any(|(flag, _, _)| flag.item == "help") { @@ -176,7 +175,7 @@ pub fn eval_call( // We pass caller_stack here with the knowledge that internal commands // are going to be specifically looking for global state in the stack // rather than any local state. - decl.run(engine_state, caller_stack, call, input) + decl.run(engine_state, caller_stack, &call.into(), input) } } @@ -208,11 +207,13 @@ fn eval_external( ) -> Result { let decl_id = engine_state .find_decl("run-external".as_bytes(), &[]) - .ok_or(ShellError::ExternalNotSupported { span: head.span })?; + .ok_or(ShellError::ExternalNotSupported { + span: head.span(&engine_state), + })?; let command = engine_state.get_decl(decl_id); - let mut call = Call::new(head.span); + let mut call = Call::new(head.span(&engine_state)); call.add_positional(head.clone()); @@ -223,7 +224,7 @@ fn eval_external( } } - command.run(engine_state, stack, &call, input) + command.run(engine_state, stack, &(&call).into(), input) } pub fn eval_expression( @@ -507,6 +508,11 @@ pub fn eval_block( block: &Block, mut input: PipelineData, ) -> Result { + // Remove once IR is the default. + if stack.use_ir { + return eval_ir_block::(engine_state, stack, block, input); + } + D::enter_block(engine_state, block); let num_pipelines = block.len(); @@ -521,7 +527,7 @@ pub fn eval_block( for (i, element) in elements.iter().enumerate() { let next = elements.get(i + 1).unwrap_or(last); - let (next_out, next_err) = next.pipe_redirection(engine_state); + let (next_out, next_err) = next.pipe_redirection(&StateWorkingSet::new(engine_state)); let (stdout, stderr) = eval_element_redirection::( engine_state, stack, @@ -679,7 +685,10 @@ impl Eval for EvalRuntime { } else if quoted { Ok(Value::string(path, span)) } else { - let cwd = engine_state.cwd(Some(stack)).unwrap_or_default(); + let cwd = engine_state + .cwd(Some(stack)) + .map(AbsolutePathBuf::into_std_path_buf) + .unwrap_or_default(); let path = expand_path_with(path, cwd, true); Ok(Value::string(path.to_string_lossy(), span)) @@ -712,7 +721,7 @@ impl Eval for EvalRuntime { args: &[ExternalArgument], _: Span, ) -> Result { - let span = head.span; + let span = head.span(&engine_state); // FIXME: protect this collect with ctrl-c eval_external(engine_state, stack, head, args, PipelineData::empty())?.into_value(span) } @@ -779,9 +788,11 @@ impl Eval for EvalRuntime { let var_info = engine_state.get_var(*var_id); if var_info.mutable { stack.add_var(*var_id, rhs); - Ok(Value::nothing(lhs.span)) + Ok(Value::nothing(lhs.span(&engine_state))) } else { - Err(ShellError::AssignmentRequiresMutableVar { lhs_span: lhs.span }) + Err(ShellError::AssignmentRequiresMutableVar { + lhs_span: lhs.span(&engine_state), + }) } } Expr::FullCellPath(cell_path) => { @@ -797,7 +808,7 @@ impl Eval for EvalRuntime { // Reject attempts to assign to the entire $env if cell_path.tail.is_empty() { return Err(ShellError::CannotReplaceEnv { - span: cell_path.head.span, + span: cell_path.head.span(&engine_state), }); } @@ -837,15 +848,21 @@ impl Eval for EvalRuntime { lhs.upsert_data_at_cell_path(&cell_path.tail, rhs)?; stack.add_var(*var_id, lhs); } - Ok(Value::nothing(cell_path.head.span)) + Ok(Value::nothing(cell_path.head.span(&engine_state))) } else { - Err(ShellError::AssignmentRequiresMutableVar { lhs_span: lhs.span }) + Err(ShellError::AssignmentRequiresMutableVar { + lhs_span: lhs.span(&engine_state), + }) } } - _ => Err(ShellError::AssignmentRequiresVar { lhs_span: lhs.span }), + _ => Err(ShellError::AssignmentRequiresVar { + lhs_span: lhs.span(&engine_state), + }), } } - _ => Err(ShellError::AssignmentRequiresVar { lhs_span: lhs.span }), + _ => Err(ShellError::AssignmentRequiresVar { + lhs_span: lhs.span(&engine_state), + }), } } @@ -882,8 +899,8 @@ impl Eval for EvalRuntime { Ok(Value::string(name, span)) } - fn unreachable(expr: &Expression) -> Result { - Ok(Value::nothing(expr.span)) + fn unreachable(engine_state: &EngineState, expr: &Expression) -> Result { + Ok(Value::nothing(expr.span(&engine_state))) } } @@ -892,7 +909,7 @@ impl Eval for EvalRuntime { /// /// An automatic environment variable cannot be assigned to by user code. /// Current there are three of them: $env.PWD, $env.FILE_PWD, $env.CURRENT_FILE -fn is_automatic_env_var(var: &str) -> bool { +pub(crate) fn is_automatic_env_var(var: &str) -> bool { let names = ["PWD", "FILE_PWD", "CURRENT_FILE"]; names.iter().any(|&name| { if cfg!(windows) { diff --git a/crates/nu-engine/src/eval_helpers.rs b/crates/nu-engine/src/eval_helpers.rs index 66bda3e0eb..65ebc6b61d 100644 --- a/crates/nu-engine/src/eval_helpers.rs +++ b/crates/nu-engine/src/eval_helpers.rs @@ -1,6 +1,6 @@ use crate::{ eval_block, eval_block_with_early_return, eval_expression, eval_expression_with_input, - eval_subexpression, + eval_ir_block, eval_subexpression, }; use nu_protocol::{ ast::{Block, Expression}, @@ -13,6 +13,10 @@ use nu_protocol::{ pub type EvalBlockFn = fn(&EngineState, &mut Stack, &Block, PipelineData) -> Result; +/// Type of eval_ir_block() function +pub type EvalIrBlockFn = + fn(&EngineState, &mut Stack, &Block, PipelineData) -> Result; + /// Type of eval_block_with_early_return() function pub type EvalBlockWithEarlyReturnFn = fn(&EngineState, &mut Stack, &Block, PipelineData) -> Result; @@ -42,6 +46,16 @@ pub fn get_eval_block(engine_state: &EngineState) -> EvalBlockFn { } } +/// Helper function to fetch `eval_ir_block()` with the correct type parameter based on whether +/// engine_state is configured with or without a debugger. +pub fn get_eval_ir_block(engine_state: &EngineState) -> EvalIrBlockFn { + if engine_state.is_debugging() { + eval_ir_block:: + } else { + eval_ir_block:: + } +} + /// Helper function to fetch `eval_block_with_early_return()` with the correct type parameter based /// on whether engine_state is configured with or without a debugger. pub fn get_eval_block_with_early_return(engine_state: &EngineState) -> EvalBlockWithEarlyReturnFn { diff --git a/crates/nu-engine/src/eval_ir.rs b/crates/nu-engine/src/eval_ir.rs new file mode 100644 index 0000000000..e2ec7ccdac --- /dev/null +++ b/crates/nu-engine/src/eval_ir.rs @@ -0,0 +1,1462 @@ +use std::{borrow::Cow, fs::File, sync::Arc}; + +use nu_path::{expand_path_with, AbsolutePathBuf}; +use nu_protocol::{ + ast::{Bits, Block, Boolean, CellPath, Comparison, Math, Operator}, + debugger::DebugContext, + engine::{Argument, Closure, EngineState, ErrorHandler, Matcher, Redirection, Stack}, + ir::{Call, DataSlice, Instruction, IrAstRef, IrBlock, Literal, RedirectMode}, + record, ByteStreamSource, DeclId, ErrSpan, Flag, IntoPipelineData, IntoSpanned, ListStream, + OutDest, PipelineData, PositionalArg, Range, Record, RegId, ShellError, Signals, Signature, + Span, Spanned, Type, Value, VarId, ENV_VARIABLE_ID, +}; +use nu_utils::IgnoreCaseExt; + +use crate::{eval::is_automatic_env_var, eval_block_with_early_return}; + +/// Evaluate the compiled representation of a [`Block`]. +pub fn eval_ir_block( + engine_state: &EngineState, + stack: &mut Stack, + block: &Block, + input: PipelineData, +) -> Result { + // Rust does not check recursion limits outside of const evaluation. + // But nu programs run in the same process as the shell. + // To prevent a stack overflow in user code from crashing the shell, + // we limit the recursion depth of function calls. + let maximum_call_stack_depth: u64 = engine_state.config.recursion_limit as u64; + if stack.recursion_count > maximum_call_stack_depth { + return Err(ShellError::RecursionLimitReached { + recursion_limit: maximum_call_stack_depth, + span: block.span, + }); + } + + if let Some(ir_block) = &block.ir_block { + D::enter_block(engine_state, block); + + let args_base = stack.arguments.get_base(); + let error_handler_base = stack.error_handlers.get_base(); + + // Allocate and initialize registers. I've found that it's not really worth trying to avoid + // the heap allocation here by reusing buffers - our allocator is fast enough + let mut registers = Vec::with_capacity(ir_block.register_count as usize); + for _ in 0..ir_block.register_count { + registers.push(PipelineData::Empty); + } + + // Initialize file storage. + let mut files = vec![None; ir_block.file_count as usize]; + + let result = eval_ir_block_impl::( + &mut EvalContext { + engine_state, + stack, + data: &ir_block.data, + block_span: &block.span, + args_base, + error_handler_base, + redirect_out: None, + redirect_err: None, + matches: vec![], + registers: &mut registers[..], + files: &mut files[..], + }, + ir_block, + input, + ); + + stack.error_handlers.leave_frame(error_handler_base); + stack.arguments.leave_frame(args_base); + + D::leave_block(engine_state, block); + + result + } else { + // FIXME blocks having IR should not be optional + Err(ShellError::GenericError { + error: "Can't evaluate block in IR mode".into(), + msg: "block is missing compiled representation".into(), + span: block.span, + help: Some("the IrBlock is probably missing due to a compilation error".into()), + inner: vec![], + }) + } +} + +/// All of the pointers necessary for evaluation +struct EvalContext<'a> { + engine_state: &'a EngineState, + stack: &'a mut Stack, + data: &'a Arc<[u8]>, + /// The span of the block + block_span: &'a Option, + /// Base index on the argument stack to reset to after a call + args_base: usize, + /// Base index on the error handler stack to reset to after a call + error_handler_base: usize, + /// State set by redirect-out + redirect_out: Option, + /// State set by redirect-err + redirect_err: Option, + /// Scratch space to use for `match` + matches: Vec<(VarId, Value)>, + /// Intermediate pipeline data storage used by instructions, indexed by RegId + registers: &'a mut [PipelineData], + /// Holds open files used by redirections + files: &'a mut [Option>], +} + +impl<'a> EvalContext<'a> { + /// Replace the contents of a register with a new value + #[inline] + fn put_reg(&mut self, reg_id: RegId, new_value: PipelineData) { + // log::trace!("{reg_id} <- {new_value:?}"); + self.registers[reg_id.0 as usize] = new_value; + } + + /// Borrow the contents of a register. + #[inline] + fn borrow_reg(&self, reg_id: RegId) -> &PipelineData { + &self.registers[reg_id.0 as usize] + } + + /// Replace the contents of a register with `Empty` and then return the value that it contained + #[inline] + fn take_reg(&mut self, reg_id: RegId) -> PipelineData { + // log::trace!("<- {reg_id}"); + std::mem::replace(&mut self.registers[reg_id.0 as usize], PipelineData::Empty) + } + + /// Clone data from a register. Must be collected first. + fn clone_reg(&mut self, reg_id: RegId, error_span: Span) -> Result { + match &self.registers[reg_id.0 as usize] { + PipelineData::Empty => Ok(PipelineData::Empty), + PipelineData::Value(val, meta) => Ok(PipelineData::Value(val.clone(), meta.clone())), + _ => Err(ShellError::IrEvalError { + msg: "Must collect to value before using instruction that clones from a register" + .into(), + span: Some(error_span), + }), + } + } + + /// Clone a value from a register. Must be collected first. + fn clone_reg_value(&mut self, reg_id: RegId, fallback_span: Span) -> Result { + match self.clone_reg(reg_id, fallback_span)? { + PipelineData::Empty => Ok(Value::nothing(fallback_span)), + PipelineData::Value(val, _) => Ok(val), + _ => unreachable!("clone_reg should never return stream data"), + } + } + + /// Take and implicitly collect a register to a value + fn collect_reg(&mut self, reg_id: RegId, fallback_span: Span) -> Result { + let data = self.take_reg(reg_id); + let span = data.span().unwrap_or(fallback_span); + data.into_value(span) + } + + /// Get a string from data or produce evaluation error if it's invalid UTF-8 + fn get_str(&self, slice: DataSlice, error_span: Span) -> Result<&'a str, ShellError> { + std::str::from_utf8(&self.data[slice]).map_err(|_| ShellError::IrEvalError { + msg: format!("data slice does not refer to valid UTF-8: {slice:?}"), + span: Some(error_span), + }) + } +} + +/// Eval an IR block on the provided slice of registers. +fn eval_ir_block_impl( + ctx: &mut EvalContext<'_>, + ir_block: &IrBlock, + input: PipelineData, +) -> Result { + if !ctx.registers.is_empty() { + ctx.registers[0] = input; + } + + // Program counter, starts at zero. + let mut pc = 0; + + while pc < ir_block.instructions.len() { + let instruction = &ir_block.instructions[pc]; + let span = &ir_block.spans[pc]; + let ast = &ir_block.ast[pc]; + log::trace!( + "{pc:-4}: {}", + instruction.display(ctx.engine_state, ctx.data) + ); + match eval_instruction::(ctx, instruction, span, ast) { + Ok(InstructionResult::Continue) => { + pc += 1; + } + Ok(InstructionResult::Branch(next_pc)) => { + pc = next_pc; + } + Ok(InstructionResult::Return(reg_id)) => { + return Ok(ctx.take_reg(reg_id)); + } + Ok(InstructionResult::ExitCode(exit_code)) => { + if let Some(error_handler) = ctx.stack.error_handlers.pop(ctx.error_handler_base) { + // If an error handler is set, branch there + prepare_error_handler(ctx, error_handler, None); + pc = error_handler.handler_index; + } else { + // If not, exit the block with the exit code + return Ok(PipelineData::new_external_stream_with_only_exit_code( + exit_code, + )); + } + } + Err( + err @ (ShellError::Return { .. } + | ShellError::Continue { .. } + | ShellError::Break { .. }), + ) => { + // These block control related errors should be passed through + return Err(err); + } + Err(err) => { + if let Some(error_handler) = ctx.stack.error_handlers.pop(ctx.error_handler_base) { + // If an error handler is set, branch there + prepare_error_handler(ctx, error_handler, Some(err.into_spanned(*span))); + pc = error_handler.handler_index; + } else { + // If not, exit the block with the error + return Err(err); + } + } + } + } + + // Fell out of the loop, without encountering a Return. + Err(ShellError::IrEvalError { + msg: format!( + "Program counter out of range (pc={pc}, len={len})", + len = ir_block.instructions.len(), + ), + span: *ctx.block_span, + }) +} + +/// Prepare the context for an error handler +fn prepare_error_handler( + ctx: &mut EvalContext<'_>, + error_handler: ErrorHandler, + error: Option>, +) { + if let Some(reg_id) = error_handler.error_register { + if let Some(error) = error { + // Create the error value and put it in the register + let value = Value::record( + record! { + "msg" => Value::string(format!("{}", error.item), error.span), + "debug" => Value::string(format!("{:?}", error.item), error.span), + "raw" => Value::error(error.item, error.span), + }, + error.span, + ); + ctx.put_reg(reg_id, PipelineData::Value(value, None)); + } else { + // Set the register to empty + ctx.put_reg(reg_id, PipelineData::Empty); + } + } +} + +/// The result of performing an instruction. Describes what should happen next +#[derive(Debug)] +enum InstructionResult { + Continue, + Branch(usize), + Return(RegId), + ExitCode(i32), +} + +/// Perform an instruction +fn eval_instruction( + ctx: &mut EvalContext<'_>, + instruction: &Instruction, + span: &Span, + ast: &Option, +) -> Result { + use self::InstructionResult::*; + + // See the docs for `Instruction` for more information on what these instructions are supposed + // to do. + match instruction { + Instruction::Unreachable => Err(ShellError::IrEvalError { + msg: "Reached unreachable code".into(), + span: Some(*span), + }), + Instruction::LoadLiteral { dst, lit } => load_literal(ctx, *dst, lit, *span), + Instruction::LoadValue { dst, val } => { + ctx.put_reg(*dst, Value::clone(val).into_pipeline_data()); + Ok(Continue) + } + Instruction::Move { dst, src } => { + let val = ctx.take_reg(*src); + ctx.put_reg(*dst, val); + Ok(Continue) + } + Instruction::Clone { dst, src } => { + let data = ctx.clone_reg(*src, *span)?; + ctx.put_reg(*dst, data); + Ok(Continue) + } + Instruction::Collect { src_dst } => { + let data = ctx.take_reg(*src_dst); + let value = collect(data, *span)?; + ctx.put_reg(*src_dst, value); + Ok(Continue) + } + Instruction::Span { src_dst } => { + let data = ctx.take_reg(*src_dst); + let spanned = data.with_span(*span); + ctx.put_reg(*src_dst, spanned); + Ok(Continue) + } + Instruction::Drop { src } => { + ctx.take_reg(*src); + Ok(Continue) + } + Instruction::Drain { src } => { + let data = ctx.take_reg(*src); + drain(ctx, data) + } + Instruction::LoadVariable { dst, var_id } => { + let value = get_var(ctx, *var_id, *span)?; + ctx.put_reg(*dst, value.into_pipeline_data()); + Ok(Continue) + } + Instruction::StoreVariable { var_id, src } => { + let value = ctx.collect_reg(*src, *span)?; + ctx.stack.add_var(*var_id, value); + Ok(Continue) + } + Instruction::LoadEnv { dst, key } => { + let key = ctx.get_str(*key, *span)?; + if let Some(value) = get_env_var_case_insensitive(ctx, key) { + let new_value = value.clone().into_pipeline_data(); + ctx.put_reg(*dst, new_value); + Ok(Continue) + } else { + // FIXME: using the same span twice, shouldn't this really be + // EnvVarNotFoundAtRuntime? There are tests that depend on CantFindColumn though... + Err(ShellError::CantFindColumn { + col_name: key.into(), + span: Some(*span), + src_span: *span, + }) + } + } + Instruction::LoadEnvOpt { dst, key } => { + let key = ctx.get_str(*key, *span)?; + let value = get_env_var_case_insensitive(ctx, key) + .cloned() + .unwrap_or(Value::nothing(*span)); + ctx.put_reg(*dst, value.into_pipeline_data()); + Ok(Continue) + } + Instruction::StoreEnv { key, src } => { + let key = ctx.get_str(*key, *span)?; + let value = ctx.collect_reg(*src, *span)?; + + let key = get_env_var_name_case_insensitive(ctx, key); + + if !is_automatic_env_var(&key) { + ctx.stack.add_env_var(key.into_owned(), value); + Ok(Continue) + } else { + Err(ShellError::AutomaticEnvVarSetManually { + envvar_name: key.into(), + span: *span, + }) + } + } + Instruction::PushPositional { src } => { + let val = ctx.collect_reg(*src, *span)?.with_span(*span); + ctx.stack.arguments.push(Argument::Positional { + span: *span, + val, + ast: ast.clone().map(|ast_ref| ast_ref.0), + }); + Ok(Continue) + } + Instruction::AppendRest { src } => { + let vals = ctx.collect_reg(*src, *span)?.with_span(*span); + ctx.stack.arguments.push(Argument::Spread { + span: *span, + vals, + ast: ast.clone().map(|ast_ref| ast_ref.0), + }); + Ok(Continue) + } + Instruction::PushFlag { name } => { + let data = ctx.data.clone(); + ctx.stack.arguments.push(Argument::Flag { + data, + name: *name, + short: DataSlice::empty(), + span: *span, + }); + Ok(Continue) + } + Instruction::PushShortFlag { short } => { + let data = ctx.data.clone(); + ctx.stack.arguments.push(Argument::Flag { + data, + name: DataSlice::empty(), + short: *short, + span: *span, + }); + Ok(Continue) + } + Instruction::PushNamed { name, src } => { + let val = ctx.collect_reg(*src, *span)?.with_span(*span); + let data = ctx.data.clone(); + ctx.stack.arguments.push(Argument::Named { + data, + name: *name, + short: DataSlice::empty(), + span: *span, + val, + ast: ast.clone().map(|ast_ref| ast_ref.0), + }); + Ok(Continue) + } + Instruction::PushShortNamed { short, src } => { + let val = ctx.collect_reg(*src, *span)?.with_span(*span); + let data = ctx.data.clone(); + ctx.stack.arguments.push(Argument::Named { + data, + name: DataSlice::empty(), + short: *short, + span: *span, + val, + ast: ast.clone().map(|ast_ref| ast_ref.0), + }); + Ok(Continue) + } + Instruction::PushParserInfo { name, info } => { + let data = ctx.data.clone(); + ctx.stack.arguments.push(Argument::ParserInfo { + data, + name: *name, + info: info.clone(), + }); + Ok(Continue) + } + Instruction::RedirectOut { mode } => { + ctx.redirect_out = eval_redirection(ctx, mode, *span, RedirectionStream::Out)?; + Ok(Continue) + } + Instruction::RedirectErr { mode } => { + ctx.redirect_err = eval_redirection(ctx, mode, *span, RedirectionStream::Err)?; + Ok(Continue) + } + Instruction::CheckErrRedirected { src } => match ctx.borrow_reg(*src) { + PipelineData::ByteStream(stream, _) + if matches!(stream.source(), ByteStreamSource::Child(_)) => + { + Ok(Continue) + } + _ => Err(ShellError::GenericError { + error: "Can't redirect stderr of internal command output".into(), + msg: "piping stderr only works on external commands".into(), + span: Some(*span), + help: None, + inner: vec![], + }), + }, + Instruction::OpenFile { + file_num, + path, + append, + } => { + let path = ctx.collect_reg(*path, *span)?; + let file = open_file(ctx, &path, *append)?; + ctx.files[*file_num as usize] = Some(file); + Ok(Continue) + } + Instruction::WriteFile { file_num, src } => { + let src = ctx.take_reg(*src); + let file = ctx + .files + .get(*file_num as usize) + .cloned() + .flatten() + .ok_or_else(|| ShellError::IrEvalError { + msg: format!("Tried to write to file #{file_num}, but it is not open"), + span: Some(*span), + })?; + let result = { + let mut stack = ctx + .stack + .push_redirection(Some(Redirection::File(file)), None); + src.write_to_out_dests(ctx.engine_state, &mut stack)? + }; + // Abort execution if there's an exit code from a failed external + drain(ctx, result) + } + Instruction::CloseFile { file_num } => { + if ctx.files[*file_num as usize].take().is_some() { + Ok(Continue) + } else { + Err(ShellError::IrEvalError { + msg: format!("Tried to close file #{file_num}, but it is not open"), + span: Some(*span), + }) + } + } + Instruction::Call { decl_id, src_dst } => { + let input = ctx.take_reg(*src_dst); + let result = eval_call::(ctx, *decl_id, *span, input)?; + ctx.put_reg(*src_dst, result); + Ok(Continue) + } + Instruction::StringAppend { src_dst, val } => { + let string_value = ctx.collect_reg(*src_dst, *span)?; + let operand_value = ctx.collect_reg(*val, *span)?; + let string_span = string_value.span(); + + let mut string = string_value.into_string()?; + let operand = if let Value::String { val, .. } = operand_value { + // Small optimization, so we don't have to copy the string *again* + val + } else { + operand_value.to_expanded_string(", ", ctx.engine_state.get_config()) + }; + string.push_str(&operand); + + let new_string_value = Value::string(string, string_span); + ctx.put_reg(*src_dst, new_string_value.into_pipeline_data()); + Ok(Continue) + } + Instruction::GlobFrom { src_dst, no_expand } => { + let string_value = ctx.collect_reg(*src_dst, *span)?; + let glob_value = if matches!(string_value, Value::Glob { .. }) { + // It already is a glob, so don't touch it. + string_value + } else { + // Treat it as a string, then cast + let string = string_value.into_string()?; + Value::glob(string, *no_expand, *span) + }; + ctx.put_reg(*src_dst, glob_value.into_pipeline_data()); + Ok(Continue) + } + Instruction::ListPush { src_dst, item } => { + let list_value = ctx.collect_reg(*src_dst, *span)?; + let item = ctx.collect_reg(*item, *span)?; + let list_span = list_value.span(); + let mut list = list_value.into_list()?; + list.push(item); + ctx.put_reg(*src_dst, Value::list(list, list_span).into_pipeline_data()); + Ok(Continue) + } + Instruction::ListSpread { src_dst, items } => { + let list_value = ctx.collect_reg(*src_dst, *span)?; + let items = ctx.collect_reg(*items, *span)?; + let list_span = list_value.span(); + let items_span = items.span(); + let mut list = list_value.into_list()?; + list.extend( + items + .into_list() + .map_err(|_| ShellError::CannotSpreadAsList { span: items_span })?, + ); + ctx.put_reg(*src_dst, Value::list(list, list_span).into_pipeline_data()); + Ok(Continue) + } + Instruction::RecordInsert { src_dst, key, val } => { + let record_value = ctx.collect_reg(*src_dst, *span)?; + let key = ctx.collect_reg(*key, *span)?; + let val = ctx.collect_reg(*val, *span)?; + let record_span = record_value.span(); + let mut record = record_value.into_record()?; + + let key = key.coerce_into_string()?; + if let Some(old_value) = record.insert(&key, val) { + return Err(ShellError::ColumnDefinedTwice { + col_name: key, + second_use: *span, + first_use: old_value.span(), + }); + } + + ctx.put_reg( + *src_dst, + Value::record(record, record_span).into_pipeline_data(), + ); + Ok(Continue) + } + Instruction::RecordSpread { src_dst, items } => { + let record_value = ctx.collect_reg(*src_dst, *span)?; + let items = ctx.collect_reg(*items, *span)?; + let record_span = record_value.span(); + let items_span = items.span(); + let mut record = record_value.into_record()?; + // Not using .extend() here because it doesn't handle duplicates + for (key, val) in items + .into_record() + .map_err(|_| ShellError::CannotSpreadAsRecord { span: items_span })? + { + if let Some(first_value) = record.insert(&key, val) { + return Err(ShellError::ColumnDefinedTwice { + col_name: key, + second_use: *span, + first_use: first_value.span(), + }); + } + } + ctx.put_reg( + *src_dst, + Value::record(record, record_span).into_pipeline_data(), + ); + Ok(Continue) + } + Instruction::Not { src_dst } => { + let bool = ctx.collect_reg(*src_dst, *span)?; + let negated = !bool.as_bool()?; + ctx.put_reg( + *src_dst, + Value::bool(negated, bool.span()).into_pipeline_data(), + ); + Ok(Continue) + } + Instruction::BinaryOp { lhs_dst, op, rhs } => binary_op(ctx, *lhs_dst, op, *rhs, *span), + Instruction::FollowCellPath { src_dst, path } => { + let data = ctx.take_reg(*src_dst); + let path = ctx.take_reg(*path); + if let PipelineData::Value(Value::CellPath { val: path, .. }, _) = path { + let value = data.follow_cell_path(&path.members, *span, true)?; + ctx.put_reg(*src_dst, value.into_pipeline_data()); + Ok(Continue) + } else if let PipelineData::Value(Value::Error { error, .. }, _) = path { + Err(*error) + } else { + Err(ShellError::TypeMismatch { + err_message: "expected cell path".into(), + span: path.span().unwrap_or(*span), + }) + } + } + Instruction::CloneCellPath { dst, src, path } => { + let value = ctx.clone_reg_value(*src, *span)?; + let path = ctx.take_reg(*path); + if let PipelineData::Value(Value::CellPath { val: path, .. }, _) = path { + // TODO: make follow_cell_path() not have to take ownership, probably using Cow + let value = value.follow_cell_path(&path.members, true)?; + ctx.put_reg(*dst, value.into_pipeline_data()); + Ok(Continue) + } else if let PipelineData::Value(Value::Error { error, .. }, _) = path { + Err(*error) + } else { + Err(ShellError::TypeMismatch { + err_message: "expected cell path".into(), + span: path.span().unwrap_or(*span), + }) + } + } + Instruction::UpsertCellPath { + src_dst, + path, + new_value, + } => { + let data = ctx.take_reg(*src_dst); + let metadata = data.metadata(); + // Change the span because we're modifying it + let mut value = data.into_value(*span)?; + let path = ctx.take_reg(*path); + let new_value = ctx.collect_reg(*new_value, *span)?; + if let PipelineData::Value(Value::CellPath { val: path, .. }, _) = path { + value.upsert_data_at_cell_path(&path.members, new_value)?; + ctx.put_reg(*src_dst, value.into_pipeline_data_with_metadata(metadata)); + Ok(Continue) + } else if let PipelineData::Value(Value::Error { error, .. }, _) = path { + Err(*error) + } else { + Err(ShellError::TypeMismatch { + err_message: "expected cell path".into(), + span: path.span().unwrap_or(*span), + }) + } + } + Instruction::Jump { index } => Ok(Branch(*index)), + Instruction::BranchIf { cond, index } => { + let data = ctx.take_reg(*cond); + let data_span = data.span(); + let val = match data { + PipelineData::Value(Value::Bool { val, .. }, _) => val, + PipelineData::Value(Value::Error { error, .. }, _) => { + return Err(*error); + } + _ => { + return Err(ShellError::TypeMismatch { + err_message: "expected bool".into(), + span: data_span.unwrap_or(*span), + }); + } + }; + if val { + Ok(Branch(*index)) + } else { + Ok(Continue) + } + } + Instruction::BranchIfEmpty { src, index } => { + let is_empty = matches!( + ctx.borrow_reg(*src), + PipelineData::Empty | PipelineData::Value(Value::Nothing { .. }, _) + ); + + if is_empty { + Ok(Branch(*index)) + } else { + Ok(Continue) + } + } + Instruction::Match { + pattern, + src, + index, + } => { + let value = ctx.clone_reg_value(*src, *span)?; + ctx.matches.clear(); + if pattern.match_value(&value, &mut ctx.matches) { + // Match succeeded: set variables and branch + for (var_id, match_value) in ctx.matches.drain(..) { + ctx.stack.add_var(var_id, match_value); + } + Ok(Branch(*index)) + } else { + // Failed to match, put back original value + ctx.matches.clear(); + Ok(Continue) + } + } + Instruction::CheckMatchGuard { src } => { + if matches!( + ctx.borrow_reg(*src), + PipelineData::Value(Value::Bool { .. }, _) + ) { + Ok(Continue) + } else { + Err(ShellError::MatchGuardNotBool { span: *span }) + } + } + Instruction::Iterate { + dst, + stream, + end_index, + } => eval_iterate(ctx, *dst, *stream, *end_index), + Instruction::OnError { index } => { + ctx.stack.error_handlers.push(ErrorHandler { + handler_index: *index, + error_register: None, + }); + Ok(Continue) + } + Instruction::OnErrorInto { index, dst } => { + ctx.stack.error_handlers.push(ErrorHandler { + handler_index: *index, + error_register: Some(*dst), + }); + Ok(Continue) + } + Instruction::PopErrorHandler => { + ctx.stack.error_handlers.pop(ctx.error_handler_base); + Ok(Continue) + } + Instruction::CheckExternalFailed { dst, src } => { + let data = ctx.take_reg(*src); + let (data, failed) = data.check_external_failed()?; + ctx.put_reg(*src, data); + ctx.put_reg(*dst, Value::bool(failed, *span).into_pipeline_data()); + Ok(Continue) + } + Instruction::ReturnEarly { src } => { + let val = ctx.collect_reg(*src, *span)?; + Err(ShellError::Return { + span: *span, + value: Box::new(val), + }) + } + Instruction::Return { src } => Ok(Return(*src)), + } +} + +/// Load a literal value into a register +fn load_literal( + ctx: &mut EvalContext<'_>, + dst: RegId, + lit: &Literal, + span: Span, +) -> Result { + let value = literal_value(ctx, lit, span)?; + ctx.put_reg(dst, PipelineData::Value(value, None)); + Ok(InstructionResult::Continue) +} + +fn literal_value( + ctx: &mut EvalContext<'_>, + lit: &Literal, + span: Span, +) -> Result { + Ok(match lit { + Literal::Bool(b) => Value::bool(*b, span), + Literal::Int(i) => Value::int(*i, span), + Literal::Float(f) => Value::float(*f, span), + Literal::Filesize(q) => Value::filesize(*q, span), + Literal::Duration(q) => Value::duration(*q, span), + Literal::Binary(bin) => Value::binary(&ctx.data[*bin], span), + Literal::Block(block_id) | Literal::RowCondition(block_id) | Literal::Closure(block_id) => { + let block = ctx.engine_state.get_block(*block_id); + let captures = block + .captures + .iter() + .map(|var_id| get_var(ctx, *var_id, span).map(|val| (*var_id, val))) + .collect::, ShellError>>()?; + Value::closure( + Closure { + block_id: *block_id, + captures, + }, + span, + ) + } + Literal::Range { + start, + step, + end, + inclusion, + } => { + let start = ctx.collect_reg(*start, span)?; + let step = ctx.collect_reg(*step, span)?; + let end = ctx.collect_reg(*end, span)?; + let range = Range::new(start, step, end, *inclusion, span)?; + Value::range(range, span) + } + Literal::List { capacity } => Value::list(Vec::with_capacity(*capacity), span), + Literal::Record { capacity } => Value::record(Record::with_capacity(*capacity), span), + Literal::Filepath { + val: path, + no_expand, + } => { + let path = ctx.get_str(*path, span)?; + if *no_expand { + Value::string(path, span) + } else { + let cwd = ctx.engine_state.cwd(Some(ctx.stack))?; + let path = expand_path_with(path, cwd, true); + + Value::string(path.to_string_lossy(), span) + } + } + Literal::Directory { + val: path, + no_expand, + } => { + let path = ctx.get_str(*path, span)?; + if path == "-" { + Value::string("-", span) + } else if *no_expand { + Value::string(path, span) + } else { + let cwd = ctx + .engine_state + .cwd(Some(ctx.stack)) + .map(AbsolutePathBuf::into_std_path_buf) + .unwrap_or_default(); + let path = expand_path_with(path, cwd, true); + + Value::string(path.to_string_lossy(), span) + } + } + Literal::GlobPattern { val, no_expand } => { + Value::glob(ctx.get_str(*val, span)?, *no_expand, span) + } + Literal::String(s) => Value::string(ctx.get_str(*s, span)?, span), + Literal::RawString(s) => Value::string(ctx.get_str(*s, span)?, span), + Literal::CellPath(path) => Value::cell_path(CellPath::clone(path), span), + Literal::Date(dt) => Value::date(**dt, span), + Literal::Nothing => Value::nothing(span), + }) +} + +fn binary_op( + ctx: &mut EvalContext<'_>, + lhs_dst: RegId, + op: &Operator, + rhs: RegId, + span: Span, +) -> Result { + let lhs_val = ctx.collect_reg(lhs_dst, span)?; + let rhs_val = ctx.collect_reg(rhs, span)?; + + // Handle binary op errors early + if let Value::Error { error, .. } = lhs_val { + return Err(*error); + } + if let Value::Error { error, .. } = rhs_val { + return Err(*error); + } + + // We only have access to one span here, but the generated code usually adds a `span` + // instruction to set the output span to the right span. + let op_span = span; + + let result = match op { + Operator::Comparison(cmp) => match cmp { + Comparison::Equal => lhs_val.eq(op_span, &rhs_val, span)?, + Comparison::NotEqual => lhs_val.ne(op_span, &rhs_val, span)?, + Comparison::LessThan => lhs_val.lt(op_span, &rhs_val, span)?, + Comparison::GreaterThan => lhs_val.gt(op_span, &rhs_val, span)?, + Comparison::LessThanOrEqual => lhs_val.lte(op_span, &rhs_val, span)?, + Comparison::GreaterThanOrEqual => lhs_val.gte(op_span, &rhs_val, span)?, + Comparison::RegexMatch => { + lhs_val.regex_match(ctx.engine_state, op_span, &rhs_val, false, span)? + } + Comparison::NotRegexMatch => { + lhs_val.regex_match(ctx.engine_state, op_span, &rhs_val, true, span)? + } + Comparison::In => lhs_val.r#in(op_span, &rhs_val, span)?, + Comparison::NotIn => lhs_val.not_in(op_span, &rhs_val, span)?, + Comparison::StartsWith => lhs_val.starts_with(op_span, &rhs_val, span)?, + Comparison::EndsWith => lhs_val.ends_with(op_span, &rhs_val, span)?, + }, + Operator::Math(mat) => match mat { + Math::Plus => lhs_val.add(op_span, &rhs_val, span)?, + Math::Append => lhs_val.append(op_span, &rhs_val, span)?, + Math::Minus => lhs_val.sub(op_span, &rhs_val, span)?, + Math::Multiply => lhs_val.mul(op_span, &rhs_val, span)?, + Math::Divide => lhs_val.div(op_span, &rhs_val, span)?, + Math::Modulo => lhs_val.modulo(op_span, &rhs_val, span)?, + Math::FloorDivision => lhs_val.floor_div(op_span, &rhs_val, span)?, + Math::Pow => lhs_val.pow(op_span, &rhs_val, span)?, + }, + Operator::Boolean(bl) => match bl { + Boolean::And => lhs_val.and(op_span, &rhs_val, span)?, + Boolean::Or => lhs_val.or(op_span, &rhs_val, span)?, + Boolean::Xor => lhs_val.xor(op_span, &rhs_val, span)?, + }, + Operator::Bits(bit) => match bit { + Bits::BitOr => lhs_val.bit_or(op_span, &rhs_val, span)?, + Bits::BitXor => lhs_val.bit_xor(op_span, &rhs_val, span)?, + Bits::BitAnd => lhs_val.bit_and(op_span, &rhs_val, span)?, + Bits::ShiftLeft => lhs_val.bit_shl(op_span, &rhs_val, span)?, + Bits::ShiftRight => lhs_val.bit_shr(op_span, &rhs_val, span)?, + }, + Operator::Assignment(_asg) => { + return Err(ShellError::IrEvalError { + msg: "can't eval assignment with the `binary-op` instruction".into(), + span: Some(span), + }) + } + }; + + ctx.put_reg(lhs_dst, PipelineData::Value(result, None)); + + Ok(InstructionResult::Continue) +} + +/// Evaluate a call +fn eval_call( + ctx: &mut EvalContext<'_>, + decl_id: DeclId, + head: Span, + input: PipelineData, +) -> Result { + let EvalContext { + engine_state, + stack: caller_stack, + args_base, + redirect_out, + redirect_err, + .. + } = ctx; + + let args_len = caller_stack.arguments.get_len(*args_base); + let decl = engine_state.get_decl(decl_id); + + // Set up redirect modes + let mut caller_stack = caller_stack.push_redirection(redirect_out.take(), redirect_err.take()); + + let result; + + if let Some(block_id) = decl.block_id() { + // If the decl is a custom command + let block = engine_state.get_block(block_id); + + // Set up a callee stack with the captures and move arguments from the stack into variables + let mut callee_stack = caller_stack.gather_captures(engine_state, &block.captures); + + gather_arguments( + engine_state, + block, + &mut caller_stack, + &mut callee_stack, + *args_base, + args_len, + head, + )?; + + // Add one to the recursion count, so we don't recurse too deep. Stack overflows are not + // recoverable in Rust. + callee_stack.recursion_count += 1; + + result = eval_block_with_early_return::(engine_state, &mut callee_stack, block, input); + + // Move environment variables back into the caller stack scope if requested to do so + if block.redirect_env { + redirect_env(engine_state, &mut caller_stack, &callee_stack); + } + } else { + // FIXME: precalculate this and save it somewhere + let span = Span::merge_many( + std::iter::once(head).chain( + caller_stack + .arguments + .get_args(*args_base, args_len) + .iter() + .flat_map(|arg| arg.span()), + ), + ); + + let call = Call { + decl_id, + head, + span, + args_base: *args_base, + args_len, + }; + + // Run the call + result = decl.run(engine_state, &mut caller_stack, &(&call).into(), input); + }; + + drop(caller_stack); + + // Important that this runs, to reset state post-call: + ctx.stack.arguments.leave_frame(ctx.args_base); + ctx.redirect_out = None; + ctx.redirect_err = None; + + result +} + +fn find_named_var_id( + sig: &Signature, + name: &[u8], + short: &[u8], + span: Span, +) -> Result { + sig.named + .iter() + .find(|n| { + if !n.long.is_empty() { + n.long.as_bytes() == name + } else { + // It's possible to only have a short name and no long name + n.short + .is_some_and(|s| s.encode_utf8(&mut [0; 4]).as_bytes() == short) + } + }) + .ok_or_else(|| ShellError::IrEvalError { + msg: format!( + "block does not have an argument named `{}`", + String::from_utf8_lossy(name) + ), + span: Some(span), + }) + .and_then(|flag| expect_named_var_id(flag, span)) +} + +fn expect_named_var_id(arg: &Flag, span: Span) -> Result { + arg.var_id.ok_or_else(|| ShellError::IrEvalError { + msg: format!( + "block signature is missing var id for named arg `{}`", + arg.long + ), + span: Some(span), + }) +} + +fn expect_positional_var_id(arg: &PositionalArg, span: Span) -> Result { + arg.var_id.ok_or_else(|| ShellError::IrEvalError { + msg: format!( + "block signature is missing var id for positional arg `{}`", + arg.name + ), + span: Some(span), + }) +} + +/// Move arguments from the stack into variables for a custom command +fn gather_arguments( + engine_state: &EngineState, + block: &Block, + caller_stack: &mut Stack, + callee_stack: &mut Stack, + args_base: usize, + args_len: usize, + call_head: Span, +) -> Result<(), ShellError> { + let mut positional_iter = block + .signature + .required_positional + .iter() + .map(|p| (p, true)) + .chain( + block + .signature + .optional_positional + .iter() + .map(|p| (p, false)), + ); + + // Arguments that didn't get consumed by required/optional + let mut rest = vec![]; + + // If we encounter a spread, all further positionals should go to rest + let mut always_spread = false; + + for arg in caller_stack.arguments.drain_args(args_base, args_len) { + match arg { + Argument::Positional { span, val, .. } => { + // Don't check next positional arg if we encountered a spread previously + let next = (!always_spread).then(|| positional_iter.next()).flatten(); + if let Some((positional_arg, required)) = next { + let var_id = expect_positional_var_id(positional_arg, span)?; + if required { + // By checking the type of the bound variable rather than converting the + // SyntaxShape here, we might be able to save some allocations and effort + let variable = engine_state.get_var(var_id); + check_type(&val, &variable.ty)?; + } + callee_stack.add_var(var_id, val); + } else { + rest.push(val); + } + } + Argument::Spread { vals, .. } => { + if let Value::List { vals, .. } = vals { + rest.extend(vals); + // All further positional args should go to spread + always_spread = true; + } else if let Value::Error { error, .. } = vals { + return Err(*error); + } else { + return Err(ShellError::CannotSpreadAsList { span: vals.span() }); + } + } + Argument::Flag { + data, + name, + short, + span, + } => { + let var_id = find_named_var_id(&block.signature, &data[name], &data[short], span)?; + callee_stack.add_var(var_id, Value::bool(true, span)) + } + Argument::Named { + data, + name, + short, + span, + val, + .. + } => { + let var_id = find_named_var_id(&block.signature, &data[name], &data[short], span)?; + callee_stack.add_var(var_id, val) + } + Argument::ParserInfo { .. } => (), + } + } + + // Add the collected rest of the arguments if a spread argument exists + if let Some(rest_arg) = &block.signature.rest_positional { + let rest_span = rest.first().map(|v| v.span()).unwrap_or(call_head); + let var_id = expect_positional_var_id(rest_arg, rest_span)?; + callee_stack.add_var(var_id, Value::list(rest, rest_span)); + } + + // Check for arguments that haven't yet been set and set them to their defaults + for (positional_arg, _) in positional_iter { + let var_id = expect_positional_var_id(positional_arg, call_head)?; + callee_stack.add_var( + var_id, + positional_arg + .default_value + .clone() + .unwrap_or(Value::nothing(call_head)), + ); + } + + for named_arg in &block.signature.named { + if let Some(var_id) = named_arg.var_id { + // For named arguments, we do this check by looking to see if the variable was set yet on + // the stack. This assumes that the stack's variables was previously empty, but that's a + // fair assumption for a brand new callee stack. + if !callee_stack.vars.iter().any(|(id, _)| *id == var_id) { + let val = if named_arg.arg.is_none() { + Value::bool(false, call_head) + } else if let Some(value) = &named_arg.default_value { + value.clone() + } else { + Value::nothing(call_head) + }; + callee_stack.add_var(var_id, val); + } + } + } + + Ok(()) +} + +/// Type check helper. Produces `CantConvert` error if `val` is not compatible with `ty`. +fn check_type(val: &Value, ty: &Type) -> Result<(), ShellError> { + if match val { + // An empty list is compatible with any list or table type + Value::List { vals, .. } if vals.is_empty() => { + matches!(ty, Type::Any | Type::List(_) | Type::Table(_)) + } + // FIXME: the allocation that might be required here is not great, it would be nice to be + // able to just directly check whether a value is compatible with a type + _ => val.get_type().is_subtype(ty), + } { + Ok(()) + } else { + Err(ShellError::CantConvert { + to_type: ty.to_string(), + from_type: val.get_type().to_string(), + span: val.span(), + help: None, + }) + } +} + +/// Get variable from [`Stack`] or [`EngineState`] +fn get_var(ctx: &EvalContext<'_>, var_id: VarId, span: Span) -> Result { + match var_id { + // $env + ENV_VARIABLE_ID => { + let env_vars = ctx.stack.get_env_vars(ctx.engine_state); + let env_columns = env_vars.keys(); + let env_values = env_vars.values(); + + let mut pairs = env_columns + .map(|x| x.to_string()) + .zip(env_values.cloned()) + .collect::>(); + + pairs.sort_by(|a, b| a.0.cmp(&b.0)); + + Ok(Value::record(pairs.into_iter().collect(), span)) + } + _ => ctx.stack.get_var(var_id, span).or_else(|err| { + // $nu is handled by getting constant + if let Some(const_val) = ctx.engine_state.get_constant(var_id).cloned() { + Ok(const_val.with_span(span)) + } else { + Err(err) + } + }), + } +} + +/// Get an environment variable, case-insensitively +fn get_env_var_case_insensitive<'a>(ctx: &'a mut EvalContext<'_>, key: &str) -> Option<&'a Value> { + // Read scopes in order + ctx.stack + .env_vars + .iter() + .rev() + .chain(std::iter::once(&ctx.engine_state.env_vars)) + .flat_map(|overlays| { + // Read overlays in order + ctx.stack + .active_overlays + .iter() + .rev() + .filter_map(|name| overlays.get(name)) + }) + .find_map(|map| { + // Use the hashmap first to try to be faster? + map.get(key).or_else(|| { + // Check to see if it exists at all in the map + map.iter() + .find_map(|(k, v)| k.eq_ignore_case(key).then_some(v)) + }) + }) +} + +/// Get the existing name of an environment variable, case-insensitively. This is used to implement +/// case preservation of environment variables, so that changing an environment variable that +/// already exists always uses the same case. +fn get_env_var_name_case_insensitive<'a>(ctx: &mut EvalContext<'_>, key: &'a str) -> Cow<'a, str> { + // Read scopes in order + ctx.stack + .env_vars + .iter() + .rev() + .chain(std::iter::once(&ctx.engine_state.env_vars)) + .flat_map(|overlays| { + // Read overlays in order + ctx.stack + .active_overlays + .iter() + .rev() + .filter_map(|name| overlays.get(name)) + }) + .find_map(|map| { + // Use the hashmap first to try to be faster? + if map.contains_key(key) { + Some(Cow::Borrowed(key)) + } else { + map.keys().find(|k| k.eq_ignore_case(key)).map(|k| { + // it exists, but with a different case + Cow::Owned(k.to_owned()) + }) + } + }) + // didn't exist. + .unwrap_or(Cow::Borrowed(key)) +} + +/// Helper to collect values into [`PipelineData`], preserving original span and metadata +fn collect(data: PipelineData, fallback_span: Span) -> Result { + let span = data.span().unwrap_or(fallback_span); + let metadata = data.metadata(); + let value = data.into_value(span)?; + Ok(PipelineData::Value(value, metadata)) +} + +/// Helper for drain behavior. Returns `Ok(ExitCode)` on failed external. +fn drain(ctx: &mut EvalContext<'_>, data: PipelineData) -> Result { + use self::InstructionResult::*; + let span = data.span().unwrap_or(Span::unknown()); + if let Some(exit_status) = data.drain()? { + ctx.stack.add_env_var( + "LAST_EXIT_CODE".into(), + Value::int(exit_status.code() as i64, span), + ); + if exit_status.code() == 0 { + Ok(Continue) + } else { + Ok(ExitCode(exit_status.code())) + } + } else { + Ok(Continue) + } +} + +enum RedirectionStream { + Out, + Err, +} + +/// Open a file for redirection +fn open_file(ctx: &EvalContext<'_>, path: &Value, append: bool) -> Result, ShellError> { + let path_expanded = + expand_path_with(path.as_str()?, ctx.engine_state.cwd(Some(ctx.stack))?, true); + let mut options = File::options(); + if append { + options.append(true); + } else { + options.write(true).truncate(true); + } + let file = options + .create(true) + .open(path_expanded) + .err_span(path.span())?; + Ok(Arc::new(file)) +} + +/// Set up a [`Redirection`] from a [`RedirectMode`] +fn eval_redirection( + ctx: &mut EvalContext<'_>, + mode: &RedirectMode, + span: Span, + which: RedirectionStream, +) -> Result, ShellError> { + match mode { + RedirectMode::Pipe => Ok(Some(Redirection::Pipe(OutDest::Pipe))), + RedirectMode::Capture => Ok(Some(Redirection::Pipe(OutDest::Capture))), + RedirectMode::Null => Ok(Some(Redirection::Pipe(OutDest::Null))), + RedirectMode::Inherit => Ok(Some(Redirection::Pipe(OutDest::Inherit))), + RedirectMode::File { file_num } => { + let file = ctx + .files + .get(*file_num as usize) + .cloned() + .flatten() + .ok_or_else(|| ShellError::IrEvalError { + msg: format!("Tried to redirect to file #{file_num}, but it is not open"), + span: Some(span), + })?; + Ok(Some(Redirection::File(file))) + } + RedirectMode::Caller => Ok(match which { + RedirectionStream::Out => ctx.stack.pipe_stdout().cloned().map(Redirection::Pipe), + RedirectionStream::Err => ctx.stack.pipe_stderr().cloned().map(Redirection::Pipe), + }), + } +} + +/// Do an `iterate` instruction. This can be called repeatedly to get more values from an iterable +fn eval_iterate( + ctx: &mut EvalContext<'_>, + dst: RegId, + stream: RegId, + end_index: usize, +) -> Result { + let mut data = ctx.take_reg(stream); + if let PipelineData::ListStream(list_stream, _) = &mut data { + // Modify the stream, taking one value off, and branching if it's empty + if let Some(val) = list_stream.next_value() { + ctx.put_reg(dst, val.into_pipeline_data()); + ctx.put_reg(stream, data); // put the stream back so it can be iterated on again + Ok(InstructionResult::Continue) + } else { + ctx.put_reg(dst, PipelineData::Empty); + Ok(InstructionResult::Branch(end_index)) + } + } else { + // Convert the PipelineData to an iterator, and wrap it in a ListStream so it can be + // iterated on + let metadata = data.metadata(); + let span = data.span().unwrap_or(Span::unknown()); + ctx.put_reg( + stream, + PipelineData::ListStream( + ListStream::new(data.into_iter(), span, Signals::EMPTY), + metadata, + ), + ); + eval_iterate(ctx, dst, stream, end_index) + } +} + +/// Redirect environment from the callee stack to the caller stack +fn redirect_env(engine_state: &EngineState, caller_stack: &mut Stack, callee_stack: &Stack) { + // TODO: make this more efficient + // Grab all environment variables from the callee + let caller_env_vars = caller_stack.get_env_var_names(engine_state); + + // remove env vars that are present in the caller but not in the callee + // (the callee hid them) + for var in caller_env_vars.iter() { + if !callee_stack.has_env_var(engine_state, var) { + caller_stack.remove_env_var(engine_state, var); + } + } + + // add new env vars from callee to caller + for (var, value) in callee_stack.get_stack_env_vars() { + caller_stack.add_env_var(var, value); + } +} diff --git a/crates/nu-engine/src/lib.rs b/crates/nu-engine/src/lib.rs index e3c8f8eede..7ed246e975 100644 --- a/crates/nu-engine/src/lib.rs +++ b/crates/nu-engine/src/lib.rs @@ -2,16 +2,19 @@ mod call_ext; mod closure_eval; pub mod column; pub mod command_prelude; +mod compile; pub mod documentation; pub mod env; mod eval; mod eval_helpers; +mod eval_ir; mod glob_from; pub mod scope; pub use call_ext::CallExt; pub use closure_eval::*; pub use column::get_columns; +pub use compile::compile; pub use documentation::get_full_help; pub use env::*; pub use eval::{ @@ -19,4 +22,5 @@ pub use eval::{ eval_expression_with_input, eval_subexpression, eval_variable, redirect_env, }; pub use eval_helpers::*; +pub use eval_ir::eval_ir_block; pub use glob_from::glob_from; diff --git a/crates/nu-explore/Cargo.toml b/crates/nu-explore/Cargo.toml index 46756e4101..6d0816ee3e 100644 --- a/crates/nu-explore/Cargo.toml +++ b/crates/nu-explore/Cargo.toml @@ -5,21 +5,21 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-explore" edition = "2021" license = "MIT" name = "nu-explore" -version = "0.93.1" +version = "0.95.1" [lib] bench = false [dependencies] -nu-protocol = { path = "../nu-protocol", version = "0.93.1" } -nu-parser = { path = "../nu-parser", version = "0.93.1" } -nu-color-config = { path = "../nu-color-config", version = "0.93.1" } -nu-engine = { path = "../nu-engine", version = "0.93.1" } -nu-table = { path = "../nu-table", version = "0.93.1" } -nu-json = { path = "../nu-json", version = "0.93.1" } -nu-utils = { path = "../nu-utils", version = "0.93.1" } +nu-protocol = { path = "../nu-protocol", version = "0.95.1" } +nu-parser = { path = "../nu-parser", version = "0.95.1" } +nu-color-config = { path = "../nu-color-config", version = "0.95.1" } +nu-engine = { path = "../nu-engine", version = "0.95.1" } +nu-table = { path = "../nu-table", version = "0.95.1" } +nu-json = { path = "../nu-json", version = "0.95.1" } +nu-utils = { path = "../nu-utils", version = "0.95.1" } nu-ansi-term = { workspace = true } -nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.93.1" } +nu-pretty-hex = { path = "../nu-pretty-hex", version = "0.95.1" } anyhow = { workspace = true } log = { workspace = true } @@ -32,4 +32,4 @@ ansi-str = { workspace = true } unicode-width = { workspace = true } lscolors = { workspace = true, default-features = false, features = [ "nu-ansi-term", -] } +] } \ No newline at end of file diff --git a/crates/nu-explore/src/commands/expand.rs b/crates/nu-explore/src/commands/expand.rs index d7d75af6c5..b748ce2b89 100644 --- a/crates/nu-explore/src/commands/expand.rs +++ b/crates/nu-explore/src/commands/expand.rs @@ -1,7 +1,7 @@ use super::ViewCommand; use crate::{ nu_common::{self, collect_input}, - views::Preview, + views::{Preview, ViewConfig}, }; use anyhow::Result; use nu_color_config::StyleComputer; @@ -43,6 +43,7 @@ impl ViewCommand for ExpandCmd { engine_state: &EngineState, stack: &mut Stack, value: Option, + _: &ViewConfig, ) -> Result { if let Some(value) = value { let value_as_string = convert_value_to_string(value, engine_state, stack)?; @@ -66,12 +67,11 @@ fn convert_value_to_string( let config = engine_state.get_config(); Ok(vals[0][0].to_abbreviated_string(config)) } else { - let ctrlc = engine_state.ctrlc.clone(); let config = engine_state.get_config(); let style_computer = StyleComputer::from_config(engine_state, stack); Ok(nu_common::try_build_table( - ctrlc, + engine_state.signals(), config, &style_computer, value, diff --git a/crates/nu-explore/src/commands/help.rs b/crates/nu-explore/src/commands/help.rs index 8be468383e..684e713939 100644 --- a/crates/nu-explore/src/commands/help.rs +++ b/crates/nu-explore/src/commands/help.rs @@ -1,5 +1,5 @@ use super::ViewCommand; -use crate::views::Preview; +use crate::views::{Preview, ViewConfig}; use anyhow::Result; use nu_ansi_term::Color; use nu_protocol::{ @@ -99,7 +99,13 @@ impl ViewCommand for HelpCmd { Ok(()) } - fn spawn(&mut self, _: &EngineState, _: &mut Stack, _: Option) -> Result { + fn spawn( + &mut self, + _: &EngineState, + _: &mut Stack, + _: Option, + _: &ViewConfig, + ) -> Result { Ok(HelpCmd::view()) } } diff --git a/crates/nu-explore/src/commands/mod.rs b/crates/nu-explore/src/commands/mod.rs index 141dc25f56..a748ddaaa3 100644 --- a/crates/nu-explore/src/commands/mod.rs +++ b/crates/nu-explore/src/commands/mod.rs @@ -1,3 +1,5 @@ +use crate::views::ViewConfig; + use super::pager::{Pager, Transition}; use anyhow::Result; use nu_protocol::{ @@ -49,5 +51,6 @@ pub trait ViewCommand { engine_state: &EngineState, stack: &mut Stack, value: Option, + config: &ViewConfig, ) -> Result; } diff --git a/crates/nu-explore/src/commands/nu.rs b/crates/nu-explore/src/commands/nu.rs index 1310dfbd1f..84e1fb9706 100644 --- a/crates/nu-explore/src/commands/nu.rs +++ b/crates/nu-explore/src/commands/nu.rs @@ -27,7 +27,7 @@ impl NuCmd { } impl ViewCommand for NuCmd { - type View = NuView<'static>; + type View = NuView; fn name(&self) -> &'static str { Self::NAME @@ -48,6 +48,7 @@ impl ViewCommand for NuCmd { engine_state: &EngineState, stack: &mut Stack, value: Option, + config: &ViewConfig, ) -> Result { let value = value.unwrap_or_default(); @@ -62,22 +63,22 @@ impl ViewCommand for NuCmd { return Ok(NuView::Preview(Preview::new(&text))); } - let mut view = RecordView::new(columns, values); + let mut view = RecordView::new(columns, values, config.explore_config.clone()); if is_record { - view.set_orientation_current(Orientation::Left); + view.set_top_layer_orientation(Orientation::Left); } Ok(NuView::Records(view)) } } -pub enum NuView<'a> { - Records(RecordView<'a>), +pub enum NuView { + Records(RecordView), Preview(Preview), } -impl View for NuView<'_> { +impl View for NuView { fn draw(&mut self, f: &mut Frame, area: Rect, cfg: ViewConfig<'_>, layout: &mut Layout) { match self { NuView::Records(v) => v.draw(f, area, cfg, layout), @@ -119,11 +120,4 @@ impl View for NuView<'_> { NuView::Preview(v) => v.exit(), } } - - fn setup(&mut self, config: ViewConfig<'_>) { - match self { - NuView::Records(v) => v.setup(config), - NuView::Preview(v) => v.setup(config), - } - } } diff --git a/crates/nu-explore/src/commands/table.rs b/crates/nu-explore/src/commands/table.rs index 7b778a8971..9ab1e39b98 100644 --- a/crates/nu-explore/src/commands/table.rs +++ b/crates/nu-explore/src/commands/table.rs @@ -1,10 +1,9 @@ use super::ViewCommand; use crate::{ nu_common::collect_input, - views::{Orientation, RecordView}, + views::{Orientation, RecordView, ViewConfig}, }; use anyhow::Result; -use nu_ansi_term::Style; use nu_protocol::{ engine::{EngineState, Stack}, Value, @@ -19,14 +18,6 @@ pub struct TableCmd { #[derive(Debug, Default, Clone)] struct TableSettings { orientation: Option, - split_line_s: Option