Public/nushell
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: Public:main
Branches Tags
Public:main
Public:feature/nu_plugin_once
Public:dependabot/cargo/chrono-tz-0.9.0
Public:revert-13291-patch/fix-header-on-separator-issue-while-truncating
Public:revert-13102-span-id-step-2-1
Public:nightly
Public:patch-release-0.92.2
Public:0.96.1
Public:0.96.0
Public:0.95.0
Public:0.94.2
Public:0.94.1
Public:0.94.0
Public:0.93.0
Public:0.92.2
Public:0.92.1
Public:0.92.0
Public:0.91.0
Public:0.90.1
Public:0.90.0
Public:0.89.0
Public:0.88.1
Public:0.88.0
Public:0.87.1
Public:0.87.0
Public:0.86.0
Public:0.85.0
Public:0.84.0
Public:0.83.1
Public:0.83.0
Public:0.82.0
Public:0.81.0
Public:0.80.0
Public:0.79.0
Public:0.78.0
Public:0.77.1
Public:0.77.0
Public:0.76.0
Public:0.75.0
Public:0.74.0
Public:0.73.0
Public:0.72.1
Public:0.72.0
Public:0.71.0
Public:0.70.0
Public:0.69.1
Public:0.69.0
Public:0.68.1
Public:0.68.0
Public:0.67.0
Public:0.66.2
Public:0.66.1
Public:0.66.0
Public:0.65.0
Public:0.64.0
Public:0.63.1
Public:0.63.0
Public:0.62.0
Public:0.61.0
Public:0.60.0
Public:0.44.0
Public:0.43.0
Public:0.42.0
Public:0.41.0
Public:0.40.0
Public:0.39.0
Public:0.38.0
Public:0.37.0
Public:0.36.0
Public:0.35.0
Public:0.34.0
Public:0.33.0
Public:0.32.0
Public:0.31.0
Public:0.30.0
Public:0.29.0
Public:0.28.0
Public:0.27.1
Public:0.27.0
Public:0.26.0
Public:0.25.1
Public:0.25.0
Public:0.24.1
Public:0.24.0
Public:0.23.0
Public:0.22.0
Public:0.21.0
Public:0.20.0
Public:0.19.0
Public:0.18.1
Public:0.17.0
Public:0.16.0
Public:0.15.1
Public:0.15.0
Public:0.14.0
Public:0.13.0
Public:0.12.0
Public:0.11.0
Public:0.10.0
Public:0.9.0
Public:0.8.0
Public:0.7.0
Public:0.6.1
Public:0.6.0
Public:0_5_0
Public:0.4.0
Public:0.3.0
Public:0.2.0
..
compare: Public:nightly
Branches Tags
Public:feature/nu_plugin_once
Public:main
Public:dependabot/cargo/chrono-tz-0.9.0
Public:revert-13291-patch/fix-header-on-separator-issue-while-truncating
Public:revert-13102-span-id-step-2-1
Public:nightly
Public:patch-release-0.92.2
Public:0.96.1
Public:0.96.0
Public:0.95.0
Public:0.94.2
Public:0.94.1
Public:0.94.0
Public:0.93.0
Public:0.92.2
Public:0.92.1
Public:0.92.0
Public:0.91.0
Public:0.90.1
Public:0.90.0
Public:0.89.0
Public:0.88.1
Public:0.88.0
Public:0.87.1
Public:0.87.0
Public:0.86.0
Public:0.85.0
Public:0.84.0
Public:0.83.1
Public:0.83.0
Public:0.82.0
Public:0.81.0
Public:0.80.0
Public:0.79.0
Public:0.78.0
Public:0.77.1
Public:0.77.0
Public:0.76.0
Public:0.75.0
Public:0.74.0
Public:0.73.0
Public:0.72.1
Public:0.72.0
Public:0.71.0
Public:0.70.0
Public:0.69.1
Public:0.69.0
Public:0.68.1
Public:0.68.0
Public:0.67.0
Public:0.66.2
Public:0.66.1
Public:0.66.0
Public:0.65.0
Public:0.64.0
Public:0.63.1
Public:0.63.0
Public:0.62.0
Public:0.61.0
Public:0.60.0
Public:0.44.0
Public:0.43.0
Public:0.42.0
Public:0.41.0
Public:0.40.0
Public:0.39.0
Public:0.38.0
Public:0.37.0
Public:0.36.0
Public:0.35.0
Public:0.34.0
Public:0.33.0
Public:0.32.0
Public:0.31.0
Public:0.30.0
Public:0.29.0
Public:0.28.0
Public:0.27.1
Public:0.27.0
Public:0.26.0
Public:0.25.1
Public:0.25.0
Public:0.24.1
Public:0.24.0
Public:0.23.0
Public:0.22.0
Public:0.21.0
Public:0.20.0
Public:0.19.0
Public:0.18.1
Public:0.17.0
Public:0.16.0
Public:0.15.1
Public:0.15.0
Public:0.14.0
Public:0.13.0
Public:0.12.0
Public:0.11.0
Public:0.10.0
Public:0.9.0
Public:0.8.0
Public:0.7.0
Public:0.6.1
Public:0.6.0
Public:0_5_0
Public:0.4.0
Public:0.3.0
Public:0.2.0

2 Commits
main ... nightly

Author SHA1 Message Date
hustcer
1433e9bed3 Try to fix nightly 2024-04-27 10:25:20 +08:00
hustcer
1e2f34ed7b Try to upgrade ubuntu to 22.04 to fix Linux releases 2024-02-28 15:08:04 +08:00
1187 changed files with 46207 additions and 50336 deletions
Show Stats Expand all files Collapse all files

15
.github/dependabot.yml vendored
View File

@ -18,21 +18,6 @@ updates:
ignore:
- dependency-name: "*"
update-types: ["version-update:semver-patch"]
groups:
# Only update polars as a whole as there are many subcrates that need to
# be updated at once. We explicitly depend on some of them, so batch their
# updates to not take up dependabot PR slots with dysfunctional PRs
polars:
patterns:
- "polars"
- "polars-*"
# uutils/coreutils also versions all their workspace crates the same at the moment
# Most of them have bleeding edge version requirements (some not)
# see: https://github.com/uutils/coreutils/blob/main/Cargo.toml
uutils:
patterns:
- "uucore"
- "uu_*"
- package-ecosystem: "github-actions"
directory: "/"
schedule:

2
.github/pull_request_template.md vendored
View File

@ -26,7 +26,7 @@ Make sure you've run and fixed any issues with these commands:
- `cargo fmt --all -- --check` to check standard code formatting (`cargo fmt --all` applies these changes)
- `cargo clippy --workspace -- -D warnings -D clippy::unwrap_used` to check that you're using the standard code style
- `cargo test --workspace` to check that all tests pass (on Windows make sure to [enable developer mode](https://learn.microsoft.com/en-us/windows/apps/get-started/developer-mode-features-and-debugging))
- `cargo run -- -c "use toolkit.nu; toolkit test stdlib"` to run the tests for the standard library
- `cargo run -- -c "use std testing; testing run-tests --path crates/nu-std"` to run the tests for the standard library
> **Note**
> from `nushell` you can also use the `toolkit` as follows

2
.github/workflows/audit.yml vendored
View File

@ -19,7 +19,7 @@ jobs:
# Prevent sudden announcement of a new advisory from failing ci:
continue-on-error: true
steps:
- uses: actions/checkout@v4.1.7
- uses: actions/checkout@v4.1.3
- uses: rustsec/audit-check@v1.4.1
with:
token: ${{ secrets.GITHUB_TOKEN }}

60
.github/workflows/ci.yml vendored
View File

@ -29,50 +29,76 @@ jobs:
# instead of 14 GB) which is too little for us right now. Revisit when `dfr` commands are
# removed and we're only building the `polars` plugin instead
platform: [windows-latest, macos-13, ubuntu-20.04]
feature: [default, dataframe]
include:
- feature: default
flags: ""
- feature: dataframe
flags: "--features=dataframe"
exclude:
- platform: windows-latest
feature: dataframe
- platform: macos-13
feature: dataframe
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v4.1.7
- uses: actions/checkout@v4.1.3
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.9.0
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
with:
rustflags: ""
- name: cargo fmt
run: cargo fmt --all -- --check
# If changing these settings also change toolkit.nu
- name: Clippy
run: cargo clippy --workspace --exclude nu_plugin_* -- $CLIPPY_OPTIONS
run: cargo clippy --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- $CLIPPY_OPTIONS
# In tests we don't have to deny unwrap
- name: Clippy of tests
run: cargo clippy --tests --workspace --exclude nu_plugin_* -- -D warnings
run: cargo clippy --tests --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings
- name: Clippy of benchmarks
run: cargo clippy --benches --workspace --exclude nu_plugin_* -- -D warnings
run: cargo clippy --benches --workspace ${{ matrix.flags }} --exclude nu_plugin_* -- -D warnings
tests:
strategy:
fail-fast: true
matrix:
platform: [windows-latest, macos-latest, ubuntu-20.04]
feature: [default, dataframe]
include:
- default-flags: ""
# linux CI cannot handle clipboard feature
- platform: ubuntu-20.04
- default-flags: ""
- platform: ubuntu-20.04
default-flags: "--no-default-features --features=default-no-clipboard"
- feature: default
flags: ""
- feature: dataframe
flags: "--features=dataframe"
exclude:
- platform: windows-latest
feature: dataframe
- platform: macos-latest
feature: dataframe
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v4.1.7
- uses: actions/checkout@v4.1.3
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.9.0
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
with:
rustflags: ""
- name: Tests
run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }}
run: cargo test --workspace --profile ci --exclude nu_plugin_* ${{ matrix.default-flags }} ${{ matrix.flags }}
- name: Check for clean repo
shell: bash
run: |
@ -95,10 +121,12 @@ jobs:
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v4.1.7
- uses: actions/checkout@v4.1.3
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.9.0
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
with:
rustflags: ""
- name: Install Nushell
run: cargo install --path . --locked --no-default-features
@ -140,16 +168,18 @@ jobs:
# Using macOS 13 runner because 14 is based on the M1 and has half as much RAM (7 GB,
# instead of 14 GB) which is too little for us right now.
#
# Failure occurring with clippy for rust 1.77.2
# Failure occuring with clippy for rust 1.77.2
platform: [windows-latest, macos-13, ubuntu-20.04]
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v4.1.7
- uses: actions/checkout@v4.1.3
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.9.0
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
with:
rustflags: ""
- name: Clippy
run: cargo clippy --package nu_plugin_* -- $CLIPPY_OPTIONS

161
.github/workflows/nightly-build.yml vendored
View File

@ -27,7 +27,7 @@ jobs:
# if: github.repository == 'nushell/nightly'
steps:
- name: Checkout
uses: actions/checkout@v4.1.7
uses: actions/checkout@v4.1.3
if: github.repository == 'nushell/nightly'
with:
ref: main
@ -36,10 +36,10 @@ jobs:
token: ${{ secrets.WORKFLOW_TOKEN }}
- name: Setup Nushell
uses: hustcer/setup-nu@v3.12
uses: hustcer/setup-nu@v3.10
if: github.repository == 'nushell/nightly'
with:
version: 0.95.0
version: 0.92.2
# Synchronize the main branch of nightly repo with the main branch of Nushell official repo
- name: Prepare for Nightly Release
@ -84,35 +84,46 @@ jobs:
include:
- target: aarch64-apple-darwin
os: macos-latest
target_rustflags: ''
- target: x86_64-apple-darwin
os: macos-latest
target_rustflags: ''
- target: x86_64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: ''
- target: x86_64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: ''
- target: aarch64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: ''
- target: aarch64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: ''
- target: x86_64-unknown-linux-gnu
os: ubuntu-22.04
os: ubuntu-latest
target_rustflags: ''
- target: x86_64-unknown-linux-musl
os: ubuntu-22.04
os: ubuntu-latest
target_rustflags: ''
- target: aarch64-unknown-linux-gnu
os: ubuntu-22.04
os: ubuntu-latest
target_rustflags: ''
- target: armv7-unknown-linux-gnueabihf
os: ubuntu-22.04
os: ubuntu-latest
target_rustflags: ''
- target: riscv64gc-unknown-linux-gnu
os: ubuntu-latest
target_rustflags: ''
runs-on: ${{matrix.os}}
steps:
- uses: actions/checkout@v4.1.7
- uses: actions/checkout@v4.1.3
with:
ref: main
fetch-depth: 0
@ -122,24 +133,26 @@ jobs:
echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.9.0
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
with:
rustflags: ''
- name: Setup Nushell
uses: hustcer/setup-nu@v3.12
uses: hustcer/setup-nu@v3.10
with:
version: 0.95.0
version: 0.92.2
- name: Release Nu Binary
id: nu
run: nu .github/workflows/release-pkg.nu
env:
RELEASE_TYPE: standard
OS: ${{ matrix.os }}
REF: ${{ github.ref }}
TARGET: ${{ matrix.target }}
_EXTRA_: ${{ matrix.extra }}
TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
- name: Create an Issue for Release Failure
if: ${{ failure() }}
@ -161,7 +174,7 @@ jobs:
# REF: https://github.com/marketplace/actions/gh-release
# Create a release only in nushell/nightly repo
- name: Publish Archive
uses: softprops/action-gh-release@v2.0.8
uses: softprops/action-gh-release@v2.0.4
if: ${{ startsWith(github.repository, 'nushell/nightly') }}
with:
prerelease: true
@ -171,6 +184,122 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
full:
name: Full
needs: prepare
strategy:
fail-fast: false
matrix:
target:
- aarch64-apple-darwin
- x86_64-apple-darwin
- x86_64-pc-windows-msvc
- aarch64-pc-windows-msvc
- x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl
- aarch64-unknown-linux-gnu
extra: ['bin']
include:
- target: aarch64-apple-darwin
os: macos-latest
target_rustflags: '--features=dataframe'
- target: x86_64-apple-darwin
os: macos-latest
target_rustflags: '--features=dataframe'
- target: x86_64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: '--features=dataframe'
- target: x86_64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: '--features=dataframe'
- target: aarch64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: '--features=dataframe'
- target: aarch64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: '--features=dataframe'
- target: x86_64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
- target: x86_64-unknown-linux-musl
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
- target: aarch64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
runs-on: ${{matrix.os}}
steps:
- uses: actions/checkout@v4.1.3
with:
ref: main
fetch-depth: 0
- name: Update Rust Toolchain Target
run: |
echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
- name: Setup Rust toolchain and cache
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
with:
rustflags: ''
- name: Setup Nushell
uses: hustcer/setup-nu@v3.10
with:
version: 0.92.2
- name: Release Nu Binary
id: nu
run: nu .github/workflows/release-pkg.nu
env:
RELEASE_TYPE: full
OS: ${{ matrix.os }}
REF: ${{ github.ref }}
TARGET: ${{ matrix.target }}
_EXTRA_: ${{ matrix.extra }}
TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
- name: Create an Issue for Release Failure
if: ${{ failure() }}
uses: JasonEtco/create-an-issue@v2.9.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
update_existing: true
search_existing: open
filename: .github/AUTO_ISSUE_TEMPLATE/nightly-build-fail.md
- name: Set Outputs of Short SHA
id: vars
run: |
echo "date=$(date -u +'%Y-%m-%d')" >> $GITHUB_OUTPUT
sha_short=$(git rev-parse --short HEAD)
echo "sha_short=${sha_short:0:7}" >> $GITHUB_OUTPUT
# REF: https://github.com/marketplace/actions/gh-release
# Create a release only in nushell/nightly repo
- name: Publish Archive
uses: softprops/action-gh-release@v2.0.4
if: ${{ startsWith(github.repository, 'nushell/nightly') }}
with:
draft: false
prerelease: true
name: Nu-nightly-${{ steps.vars.outputs.date }}-${{ steps.vars.outputs.sha_short }}
tag_name: nightly-${{ steps.vars.outputs.sha_short }}
body: |
This is a NIGHTLY build of Nushell.
It is NOT recommended for production use.
files: ${{ steps.nu.outputs.archive }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
cleanup:
name: Cleanup
# Should only run in nushell/nightly repo
@ -181,14 +310,14 @@ jobs:
- name: Waiting for Release
run: sleep 1800
- uses: actions/checkout@v4.1.7
- uses: actions/checkout@v4.1.3
with:
ref: main
- name: Setup Nushell
uses: hustcer/setup-nu@v3.12
uses: hustcer/setup-nu@v3.10
with:
version: 0.95.0
version: 0.92.2
# Keep the last a few releases
- name: Delete Older Releases

93
.github/workflows/release-pkg.nu vendored
View File

@ -9,6 +9,7 @@
# Instructions for manually creating an MSI for Winget Releases when they fail
# Added 2022-11-29 when Windows packaging wouldn't work
# Updated again on 2023-02-23 because msis are still failing validation
# Update on 2023-10-18 to use RELEASE_TYPE env var to determine if full or not
# To run this manual for windows here are the steps I take
# checkout the release you want to publish
# 1. git checkout 0.86.0
@ -16,26 +17,28 @@
# 2. $env:CARGO_TARGET_DIR = ""
# 2. hide-env CARGO_TARGET_DIR
# 3. $env.TARGET = 'x86_64-pc-windows-msvc'
# 4. $env.GITHUB_WORKSPACE = 'D:\nushell'
# 5. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt'
# 6. $env.OS = 'windows-latest'
# 4. $env.TARGET_RUSTFLAGS = ''
# 5. $env.GITHUB_WORKSPACE = 'D:\nushell'
# 6. $env.GITHUB_OUTPUT = 'D:\nushell\output\out.txt'
# 7. $env.OS = 'windows-latest'
# 8. $env.RELEASE_TYPE = '' # There is full and '' for normal releases
# make sure 7z.exe is in your path https://www.7-zip.org/download.html
# 7. $env.Path = ($env.Path | append 'c:\apps\7-zip')
# 9. $env.Path = ($env.Path | append 'c:\apps\7-zip')
# make sure aria2c.exe is in your path https://github.com/aria2/aria2
# 8. $env.Path = ($env.Path | append 'c:\path\to\aria2c')
# 10. $env.Path = ($env.Path | append 'c:\path\to\aria2c')
# make sure you have the wixtools installed https://wixtoolset.org/
# 9. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools')
# 11. $env.Path = ($env.Path | append 'C:\Users\dschroeder\AppData\Local\tauri\WixTools')
# You need to run the release-pkg twice. The first pass, with _EXTRA_ as 'bin', makes the output
# folder and builds everything. The second pass, that generates the msi file, with _EXTRA_ as 'msi'
# 10. $env._EXTRA_ = 'bin'
# 11. source .github\workflows\release-pkg.nu
# 12. cd ..
# 13. $env._EXTRA_ = 'msi'
# 14. source .github\workflows\release-pkg.nu
# 12. $env._EXTRA_ = 'bin'
# 13. source .github\workflows\release-pkg.nu
# 14. cd ..
# 15. $env._EXTRA_ = 'msi'
# 16. source .github\workflows\release-pkg.nu
# After msi is generated, you have to update winget-pkgs repo, you'll need to patch the release
# by deleting the existing msi and uploading this new msi. Then you'll need to update the hash
# on the winget-pkgs PR. To generate the hash, run this command
# 15. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256
# 17. open target\wix\nu-0.74.0-x86_64-pc-windows-msvc.msi | hash sha256
# Then, just take the output and put it in the winget-pkgs PR for the hash on the msi
@ -45,15 +48,31 @@ let os = $env.OS
let target = $env.TARGET
# Repo source dir like `/home/runner/work/nushell/nushell`
let src = $env.GITHUB_WORKSPACE
let flags = $env.TARGET_RUSTFLAGS
let dist = $'($env.GITHUB_WORKSPACE)/output'
let version = (open Cargo.toml | get package.version)
print $'Debugging info:'
print { version: $version, bin: $bin, os: $os, target: $target, src: $src, dist: $dist }; hr-line -b
print { version: $version, bin: $bin, os: $os, releaseType: $env.RELEASE_TYPE, target: $target, src: $src, flags: $flags, dist: $dist }; hr-line -b
# Rename the full release name so that we won't break the existing scripts for standard release downloading, such as:
# curl -s https://api.github.com/repos/chmln/sd/releases/latest | grep browser_download_url | cut -d '"' -f 4 | grep x86_64-unknown-linux-musl
const FULL_RLS_NAMING = {
x86_64-apple-darwin: 'x86_64-darwin-full',
aarch64-apple-darwin: 'aarch64-darwin-full',
x86_64-unknown-linux-gnu: 'x86_64-linux-gnu-full',
x86_64-pc-windows-msvc: 'x86_64-windows-msvc-full',
x86_64-unknown-linux-musl: 'x86_64-linux-musl-full',
aarch64-unknown-linux-gnu: 'aarch64-linux-gnu-full',
aarch64-pc-windows-msvc: 'aarch64-windows-msvc-full',
riscv64gc-unknown-linux-gnu: 'riscv64-linux-gnu-full',
armv7-unknown-linux-gnueabihf: 'armv7-linux-gnueabihf-full',
}
# $env
let USE_UBUNTU = $os starts-with ubuntu
let FULL_NAME = $FULL_RLS_NAMING | get -i $target | default 'unknown-target-full'
print $'(char nl)Packaging ($bin) v($version) for ($target) in ($src)...'; hr-line -b
if not ('Cargo.lock' | path exists) { cargo generate-lockfile }
@ -72,23 +91,23 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
'aarch64-unknown-linux-gnu' => {
sudo apt-get install gcc-aarch64-linux-gnu -y
$env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = 'aarch64-linux-gnu-gcc'
cargo-build-nu
cargo-build-nu $flags
}
'riscv64gc-unknown-linux-gnu' => {
sudo apt-get install gcc-riscv64-linux-gnu -y
$env.CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER = 'riscv64-linux-gnu-gcc'
cargo-build-nu
cargo-build-nu $flags
}
'armv7-unknown-linux-gnueabihf' => {
sudo apt-get install pkg-config gcc-arm-linux-gnueabihf -y
$env.CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER = 'arm-linux-gnueabihf-gcc'
cargo-build-nu
cargo-build-nu $flags
}
_ => {
# musl-tools to fix 'Failed to find tool. Is `musl-gcc` installed?'
# Actually just for x86_64-unknown-linux-musl target
if $USE_UBUNTU { sudo apt install musl-tools -y }
cargo-build-nu
cargo-build-nu $flags
}
}
}
@ -97,7 +116,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
# Build for Windows without static-link-openssl feature
# ----------------------------------------------------------------------------
if $os in ['windows-latest'] {
cargo-build-nu
cargo-build-nu $flags
}
# ----------------------------------------------------------------------------
@ -115,15 +134,9 @@ print $'(char nl)All executable files:'; hr-line
print (ls -f ($executable | into glob)); sleep 1sec
print $'(char nl)Copying release files...'; hr-line
"To use the included Nushell plugins, register the binaries with the `plugin add` command to tell Nu where to find the plugin.
Then you can use `plugin use` to load the plugin into your session.
For example:
"To use Nu plugins, use the register command to tell Nu where to find the plugin. For example:
> plugin add ./nu_plugin_query
> plugin use query
For more information, refer to https://www.nushell.sh/book/plugins.html
" | save $'($dist)/README.txt' -f
> register ./nu_plugin_query" | save $'($dist)/README.txt' -f
[LICENSE ...(glob $executable)] | each {|it| cp -rv $it $dist } | flatten
print $'(char nl)Check binary release version detail:'; hr-line
@ -143,7 +156,7 @@ cd $dist; print $'(char nl)Creating release archive...'; hr-line
if $os in ['macos-latest'] or $USE_UBUNTU {
let files = (ls | get name)
let dest = $'($bin)-($version)-($target)'
let dest = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' }
let archive = $'($dist)/($dest).tar.gz'
mkdir $dest
@ -158,15 +171,11 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
} else if $os == 'windows-latest' {
let releaseStem = $'($bin)-($version)-($target)'
let releaseStem = if $env.RELEASE_TYPE == 'full' { $'($bin)-($version)-($FULL_NAME)' } else { $'($bin)-($version)-($target)' }
print $'(char nl)Download less related stuffs...'; hr-line
# todo: less-v661 is out but is released as a zip file. maybe we should switch to that and extract it?
aria2c https://github.com/jftuga/less-Windows/releases/download/less-v608/less.exe -o less.exe
# the below was renamed because it was failing to download for darren. it should work but it wasn't
# todo: maybe we should get rid of this aria2c dependency and just use http get?
#aria2c https://raw.githubusercontent.com/jftuga/less-Windows/master/LICENSE -o LICENSE-for-less.txt
aria2c https://github.com/jftuga/less-Windows/blob/master/LICENSE -o LICENSE-for-less.txt
aria2c https://raw.githubusercontent.com/jftuga/less-Windows/master/LICENSE -o LICENSE-for-less.txt
# Create Windows msi release package
if (get-env _EXTRA_) == 'msi' {
@ -177,7 +186,7 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
# Wix need the binaries be stored in target/release/
cp -r ($'($dist)/*' | into glob) target/release/
ls target/release/* | print
cargo install cargo-wix --version 0.3.8
cargo install cargo-wix --version 0.3.4
cargo wix --no-build --nocapture --package nu --output $wixRelease
# Workaround for https://github.com/softprops/action-gh-release/issues/280
let archive = ($wixRelease | str replace --all '\' '/')
@ -199,11 +208,19 @@ if $os in ['macos-latest'] or $USE_UBUNTU {
}
}
def 'cargo-build-nu' [] {
if $os == 'windows-latest' {
cargo build --release --all --target $target
def 'cargo-build-nu' [ options: string ] {
if ($options | str trim | is-empty) {
if $os == 'windows-latest' {
cargo build --release --all --target $target
} else {
cargo build --release --all --target $target --features=static-link-openssl
}
} else {
cargo build --release --all --target $target --features=static-link-openssl
if $os == 'windows-latest' {
cargo build --release --all --target $target $options
} else {
cargo build --release --all --target $target --features=static-link-openssl $options
}
}
}

123
.github/workflows/release.yml vendored
View File

@ -34,64 +34,167 @@ jobs:
include:
- target: aarch64-apple-darwin
os: macos-latest
target_rustflags: ''
- target: x86_64-apple-darwin
os: macos-latest
target_rustflags: ''
- target: x86_64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: ''
- target: x86_64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: ''
- target: aarch64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: ''
- target: aarch64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: ''
- target: x86_64-unknown-linux-gnu
os: ubuntu-22.04
os: ubuntu-latest
target_rustflags: ''
- target: x86_64-unknown-linux-musl
os: ubuntu-22.04
os: ubuntu-latest
target_rustflags: ''
- target: aarch64-unknown-linux-gnu
os: ubuntu-22.04
os: ubuntu-latest
target_rustflags: ''
- target: armv7-unknown-linux-gnueabihf
os: ubuntu-22.04
os: ubuntu-latest
target_rustflags: ''
- target: riscv64gc-unknown-linux-gnu
os: ubuntu-latest
target_rustflags: ''
runs-on: ${{matrix.os}}
steps:
- uses: actions/checkout@v4.1.7
- uses: actions/checkout@v4.1.3
- name: Update Rust Toolchain Target
run: |
echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
- name: Setup Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1.9.0
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
with:
cache: false
rustflags: ''
- name: Setup Nushell
uses: hustcer/setup-nu@v3.12
uses: hustcer/setup-nu@v3.10
with:
version: 0.95.0
version: 0.92.2
- name: Release Nu Binary
id: nu
run: nu .github/workflows/release-pkg.nu
env:
RELEASE_TYPE: standard
OS: ${{ matrix.os }}
REF: ${{ github.ref }}
TARGET: ${{ matrix.target }}
_EXTRA_: ${{ matrix.extra }}
TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
# REF: https://github.com/marketplace/actions/gh-release
- name: Publish Archive
uses: softprops/action-gh-release@v2.0.8
uses: softprops/action-gh-release@v2.0.4
if: ${{ startsWith(github.ref, 'refs/tags/') }}
with:
draft: true
files: ${{ steps.nu.outputs.archive }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
full:
name: Full
strategy:
fail-fast: false
matrix:
target:
- aarch64-apple-darwin
- x86_64-apple-darwin
- x86_64-pc-windows-msvc
- aarch64-pc-windows-msvc
- x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl
- aarch64-unknown-linux-gnu
extra: ['bin']
include:
- target: aarch64-apple-darwin
os: macos-latest
target_rustflags: '--features=dataframe'
- target: x86_64-apple-darwin
os: macos-latest
target_rustflags: '--features=dataframe'
- target: x86_64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: '--features=dataframe'
- target: x86_64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: '--features=dataframe'
- target: aarch64-pc-windows-msvc
extra: 'bin'
os: windows-latest
target_rustflags: '--features=dataframe'
- target: aarch64-pc-windows-msvc
extra: msi
os: windows-latest
target_rustflags: '--features=dataframe'
- target: x86_64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
- target: x86_64-unknown-linux-musl
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
- target: aarch64-unknown-linux-gnu
os: ubuntu-20.04
target_rustflags: '--features=dataframe'
runs-on: ${{matrix.os}}
steps:
- uses: actions/checkout@v4.1.3
- name: Update Rust Toolchain Target
run: |
echo "targets = ['${{matrix.target}}']" >> rust-toolchain.toml
- name: Setup Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1.8.0
# WARN: Keep the rustflags to prevent from the winget submission error: `CAQuietExec: Error 0xc0000135`
with:
cache: false
rustflags: ''
- name: Setup Nushell
uses: hustcer/setup-nu@v3.10
with:
version: 0.92.2
- name: Release Nu Binary
id: nu
run: nu .github/workflows/release-pkg.nu
env:
RELEASE_TYPE: full
OS: ${{ matrix.os }}
REF: ${{ github.ref }}
TARGET: ${{ matrix.target }}
_EXTRA_: ${{ matrix.extra }}
TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }}
# REF: https://github.com/marketplace/actions/gh-release
- name: Publish Archive
uses: softprops/action-gh-release@v2.0.4
if: ${{ startsWith(github.ref, 'refs/tags/') }}
with:
draft: true

4
.github/workflows/typos.yml vendored
View File

@ -7,7 +7,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout Actions Repository
uses: actions/checkout@v4.1.7
uses: actions/checkout@v4.1.3
- name: Check spelling
uses: crate-ci/typos@v1.23.5
uses: crate-ci/typos@v1.20.10

26
CITATION.cff
View File

@ -1,26 +0,0 @@
cff-version: 1.2.0
title: 'Nushell'
message: >-
If you use this software and wish to cite it,
you can use the metadata from this file.
type: software
authors:
- name: "The Nushell Project Team"
identifiers:
- type: url
value: 'https://github.com/nushell/nushell'
description: Repository
repository-code: 'https://github.com/nushell/nushell'
url: 'https://www.nushell.sh/'
abstract: >-
The goal of the Nushell project is to take the Unix
philosophy of shells, where pipes connect simple commands
together, and bring it to the modern style of development.
Thus, rather than being either a shell, or a programming
language, Nushell connects both by bringing a rich
programming language and a full-featured shell together
into one package.
keywords:
- nushell
- shell
license: MIT

13
CONTRIBUTING.md
View File

@ -55,6 +55,7 @@ It is good practice to cover your changes with a test. Also, try to think about
Tests can be found in different places:
* `/tests`
* `src/tests`
* command examples
* crate-specific tests
@ -71,6 +72,11 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref
cargo run
```
- Build and run with dataframe support.
```nushell
cargo run --features=dataframe
```
- Run Clippy on Nushell:
```nushell
@ -88,6 +94,11 @@ Read cargo's documentation for more details: https://doc.rust-lang.org/cargo/ref
cargo test --workspace
```
along with dataframe tests
```nushell
cargo test --workspace --features=dataframe
```
or via the `toolkit.nu` command:
```nushell
use toolkit.nu test
@ -230,7 +241,7 @@ You can help us to make the review process a smooth experience:
- Choose what simplifies having confidence in the conflict resolution and the review. **Merge commits in your branch are OK** in the squash model.
- Feel free to notify your reviewers or affected PR authors if your change might cause larger conflicts with another change.
- During the rollup of multiple PRs, we may choose to resolve merge conflicts and CI failures ourselves. (Allow maintainers to push to your branch to enable us to do this quickly.)
## License
We use the [MIT License](https://github.com/nushell/nushell/blob/main/LICENSE) in all of our Nushell projects. If you are including or referencing a crate that uses the [GPL License](https://www.gnu.org/licenses/gpl-3.0.en.html#license-text) unfortunately we will not be able to accept your PR.

1578
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

110
Cargo.toml
View File

@ -10,8 +10,8 @@ homepage = "https://www.nushell.sh"
license = "MIT"
name = "nu"
repository = "https://github.com/nushell/nushell"
rust-version = "1.78.0"
version = "0.96.2"
rust-version = "1.77.2"
version = "0.92.3"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@ -32,6 +32,7 @@ members = [
"crates/nu-cmd-extra",
"crates/nu-cmd-lang",
"crates/nu-cmd-plugin",
"crates/nu-cmd-dataframe",
"crates/nu-command",
"crates/nu-color-config",
"crates/nu-explore",
@ -39,11 +40,7 @@ members = [
"crates/nu-lsp",
"crates/nu-pretty-hex",
"crates/nu-protocol",
"crates/nu-derive-value",
"crates/nu-plugin",
"crates/nu-plugin-core",
"crates/nu-plugin-engine",
"crates/nu-plugin-protocol",
"crates/nu-plugin-test-support",
"crates/nu_plugin_inc",
"crates/nu_plugin_gstat",
@ -64,27 +61,23 @@ members = [
[workspace.dependencies]
alphanumeric-sort = "1.5"
ansi-str = "0.8"
anyhow = "1.0.82"
base64 = "0.22.1"
base64 = "0.22"
bracoxide = "0.1.2"
brotli = "5.0"
byteorder = "1.5"
bytesize = "1.3"
calamine = "0.24.0"
chardetng = "0.1.17"
chrono = { default-features = false, version = "0.4.34" }
chrono = { default-features = false, version = "0.4" }
chrono-humanize = "0.2.3"
chrono-tz = "0.8"
convert_case = "0.6"
crossbeam-channel = "0.5.8"
crossterm = "0.27"
csv = "1.3"
ctrlc = "3.4"
deunicode = "1.6.0"
dialoguer = { default-features = false, version = "0.11" }
digest = { default-features = false, version = "0.10" }
dirs = "5.0"
dirs-sys = "0.4"
dirs-next = "2.0"
dtparse = "2.0"
encoding_rs = "0.8"
fancy-regex = "0.13"
@ -97,7 +90,6 @@ heck = "0.5.0"
human-date-parser = "0.1.1"
indexmap = "2.2"
indicatif = "0.17"
interprocess = "2.2.0"
is_executable = "1.0"
itertools = "0.12"
libc = "0.2"
@ -116,37 +108,34 @@ mockito = { version = "1.4", default-features = false }
native-tls = "0.2"
nix = { version = "0.28", default-features = false }
notify-debouncer-full = { version = "0.3", default-features = false }
nu-ansi-term = "0.50.1"
nu-ansi-term = "0.50.0"
num-format = "0.4"
num-traits = "0.2"
omnipath = "0.1"
once_cell = "1.18"
open = "5.3"
os_pipe = { version = "1.2", features = ["io_safety"] }
open = "5.1"
os_pipe = "1.1"
pathdiff = "0.2"
percent-encoding = "2"
pretty_assertions = "1.4"
print-positions = "0.6"
proc-macro-error = { version = "1.0", default-features = false }
proc-macro2 = "1.0"
procfs = "0.16.0"
pwd = "1.3"
quick-xml = "0.31.0"
quickcheck = "1.0"
quickcheck_macros = "1.0"
quote = "1.0"
rand = "0.8"
ratatui = "0.26"
rayon = "1.10"
reedline = "0.33.0"
reedline = "0.31.0"
regex = "1.9.5"
rmp = "0.8"
rmp-serde = "1.3"
rmp-serde = "1.2"
ropey = "1.6.1"
roxmltree = "0.19"
rstest = { version = "0.18", default-features = false }
rusqlite = "0.31"
rust-embed = "8.5.0"
rust-embed = "8.3.0"
same-file = "1.0"
serde = { version = "1.0", default-features = false }
serde_json = "1.0"
@ -154,7 +143,6 @@ serde_urlencoded = "0.7.1"
serde_yaml = "0.9"
sha2 = "0.10"
strip-ansi-escapes = "0.2.0"
syn = "2.0"
sysinfo = "0.30"
tabled = { version = "0.14.0", default-features = false }
tempfile = "3.10"
@ -165,48 +153,50 @@ trash = "3.3"
umask = "2.1"
unicode-segmentation = "1.11"
unicode-width = "0.1"
ureq = { version = "2.10", default-features = false }
ureq = { version = "2.9", default-features = false }
url = "2.2"
uu_cp = "0.0.27"
uu_mkdir = "0.0.27"
uu_mktemp = "0.0.27"
uu_mv = "0.0.27"
uu_whoami = "0.0.27"
uu_uname = "0.0.27"
uucore = "0.0.27"
uuid = "1.10.0"
uu_cp = "0.0.25"
uu_mkdir = "0.0.25"
uu_mktemp = "0.0.25"
uu_mv = "0.0.25"
uu_whoami = "0.0.25"
uu_uname = "0.0.25"
uucore = "0.0.25"
uuid = "1.8.0"
v_htmlescape = "0.15.0"
wax = "0.6"
which = "6.0.0"
windows = "0.54"
windows-sys = "0.48"
winreg = "0.52"
[dependencies]
nu-cli = { path = "./crates/nu-cli", version = "0.96.2" }
nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.96.2" }
nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.96.2" }
nu-cmd-plugin = { path = "./crates/nu-cmd-plugin", version = "0.96.2", optional = true }
nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.96.2" }
nu-command = { path = "./crates/nu-command", version = "0.96.2" }
nu-engine = { path = "./crates/nu-engine", version = "0.96.2" }
nu-explore = { path = "./crates/nu-explore", version = "0.96.2" }
nu-lsp = { path = "./crates/nu-lsp/", version = "0.96.2" }
nu-parser = { path = "./crates/nu-parser", version = "0.96.2" }
nu-path = { path = "./crates/nu-path", version = "0.96.2" }
nu-plugin-engine = { path = "./crates/nu-plugin-engine", optional = true, version = "0.96.2" }
nu-protocol = { path = "./crates/nu-protocol", version = "0.96.2" }
nu-std = { path = "./crates/nu-std", version = "0.96.2" }
nu-system = { path = "./crates/nu-system", version = "0.96.2" }
nu-utils = { path = "./crates/nu-utils", version = "0.96.2" }
nu-cli = { path = "./crates/nu-cli", version = "0.92.3" }
nu-cmd-base = { path = "./crates/nu-cmd-base", version = "0.92.3" }
nu-cmd-lang = { path = "./crates/nu-cmd-lang", version = "0.92.3" }
nu-cmd-plugin = { path = "./crates/nu-cmd-plugin", version = "0.92.3", optional = true }
nu-cmd-dataframe = { path = "./crates/nu-cmd-dataframe", version = "0.92.3", features = [
"dataframe",
], optional = true }
nu-cmd-extra = { path = "./crates/nu-cmd-extra", version = "0.92.3" }
nu-command = { path = "./crates/nu-command", version = "0.92.3" }
nu-engine = { path = "./crates/nu-engine", version = "0.92.3" }
nu-explore = { path = "./crates/nu-explore", version = "0.92.3" }
nu-lsp = { path = "./crates/nu-lsp/", version = "0.92.3" }
nu-parser = { path = "./crates/nu-parser", version = "0.92.3" }
nu-path = { path = "./crates/nu-path", version = "0.92.3" }
nu-plugin = { path = "./crates/nu-plugin", optional = true, version = "0.92.3" }
nu-protocol = { path = "./crates/nu-protocol", version = "0.92.3" }
nu-std = { path = "./crates/nu-std", version = "0.92.3" }
nu-system = { path = "./crates/nu-system", version = "0.92.3" }
nu-utils = { path = "./crates/nu-utils", version = "0.92.3" }
reedline = { workspace = true, features = ["bashisms", "sqlite"] }
crossterm = { workspace = true }
ctrlc = { workspace = true }
dirs = { workspace = true }
log = { workspace = true }
miette = { workspace = true, features = ["fancy-no-backtrace", "fancy"] }
mimalloc = { version = "0.1.42", default-features = false, optional = true }
mimalloc = { version = "0.1.37", default-features = false, optional = true }
serde_json = { workspace = true }
simplelog = "0.12"
time = "0.3"
@ -227,21 +217,18 @@ nix = { workspace = true, default-features = false, features = [
] }
[dev-dependencies]
nu-test-support = { path = "./crates/nu-test-support", version = "0.96.2" }
nu-plugin-protocol = { path = "./crates/nu-plugin-protocol", version = "0.96.2" }
nu-plugin-core = { path = "./crates/nu-plugin-core", version = "0.96.2" }
nu-test-support = { path = "./crates/nu-test-support", version = "0.92.3" }
assert_cmd = "2.0"
dirs = { workspace = true }
tango-bench = "0.5"
dirs-next = { workspace = true }
divan = "0.1.14"
pretty_assertions = { workspace = true }
regex = { workspace = true }
rstest = { workspace = true, default-features = false }
serial_test = "3.1"
tempfile = { workspace = true }
[features]
plugin = [
"nu-plugin-engine",
"nu-plugin",
"nu-cmd-plugin",
"nu-cli/plugin",
"nu-parser/plugin",
@ -254,6 +241,7 @@ default = ["default-no-clipboard", "system-clipboard"]
# See https://github.com/nushell/nushell/pull/11535
default-no-clipboard = [
"plugin",
"which-support",
"trash-support",
"sqlite",
"mimalloc",
@ -273,8 +261,12 @@ system-clipboard = [
]
# Stable (Default)
which-support = ["nu-command/which-support", "nu-cmd-lang/which-support"]
trash-support = ["nu-command/trash-support", "nu-cmd-lang/trash-support"]
# Dataframe feature for nushell
dataframe = ["dep:nu-cmd-dataframe", "nu-cmd-lang/dataframe"]
# SQLite commands for nushell
sqlite = ["nu-command/sqlite", "nu-cmd-lang/sqlite"]

3
README.md
View File

@ -52,7 +52,7 @@ To use `Nu` in GitHub Action, check [setup-nu](https://github.com/marketplace/ac
Detailed installation instructions can be found in the [installation chapter of the book](https://www.nushell.sh/book/installation.html). Nu is available via many package managers:
[![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg?columns=3)](https://repology.org/project/nushell/versions)
[![Packaging status](https://repology.org/badge/vertical-allrepos/nushell.svg)](https://repology.org/project/nushell/versions)
For details about which platforms the Nushell team actively supports, see [our platform support policy](devdocs/PLATFORM_SUPPORT.md).
@ -222,7 +222,6 @@ Please submit an issue or PR to be added to this list.
- [clap](https://github.com/clap-rs/clap/tree/master/clap_complete_nushell)
- [Dorothy](http://github.com/bevry/dorothy)
- [Direnv](https://github.com/direnv/direnv/blob/master/docs/hook.md#nushell)
- [x-cmd](https://x-cmd.com/mod/nu)
## Contributing

29
SECURITY.md
View File

@ -1,29 +0,0 @@
# Security Policy
As a shell and programming language Nushell provides you with great powers and the potential to do dangerous things to your computer and data. Whenever there is a risk that a malicious actor can abuse a bug or a violation of documented behavior/assumptions in Nushell to harm you this is a *security* risk.
We want to fix those issues without exposing our users to unnecessary risk. Thus we want to explain our security policy.
Additional issues may be part of *safety* where the behavior of Nushell as designed and implemented can cause unintended harm or a bug causes damage without the involvement of a third party.
## Supported Versions
As Nushell is still under very active pre-stable development, the only version the core team prioritizes for security and safety fixes is the [most recent version as published on GitHub](https://github.com/nushell/nushell/releases/latest).
Only if you provide a strong reasoning and the necessary resources, will we consider blessing a backported fix with an official patch release for a previous version.
## Reporting a Vulnerability
If you suspect that a bug or behavior of Nushell can affect security or may be potentially exploitable, please report the issue to us in private.
Either reach out to the core team on [our Discord server](https://discord.gg/NtAbbGn) to arrange a private channel or use the [GitHub vulnerability reporting form](https://github.com/nushell/nushell/security/advisories/new).
Please try to answer the following questions:
- How can we reach you for further questions?
- What is the bug? Which system of Nushell may be affected?
- Do you have proof-of-concept for a potential exploit or have you observed an exploit in the wild?
- What is your assessment of the severity based on what could be impacted should the bug be exploited?
- Are additional people aware of the issue or deserve credit for identifying the issue?
We will try to get back to you within a week with:
- acknowledging the receipt of the report
- an initial plan of how we want to address this including the primary points of contact for further communication
- our preliminary assessment of how severe we judge the issue
- a proposal for how we can coordinate responsible disclosure (e.g. how we ship the bugfix, if we need to coordinate with distribution maintainers, when you can release a blog post if you want to etc.)
For purely *safety* related issues where the impact is severe by direct user action instead of malicious input or third parties, feel free to open a regular issue. If we deem that there may be an additional *security* risk on a *safety* issue we may continue discussions in a restricted forum.

813
benches/benchmarks.rs
View File

@ -1,42 +1,96 @@
use nu_cli::{eval_source, evaluate_commands};
use nu_plugin_core::{Encoder, EncodingType};
use nu_plugin_protocol::{PluginCallResponse, PluginOutput};
use nu_parser::parse;
use nu_plugin::{Encoder, EncodingType, PluginCallResponse, PluginOutput};
use nu_protocol::{
engine::{EngineState, Stack},
PipelineData, Signals, Span, Spanned, Value,
eval_const::create_nu_constant,
PipelineData, Span, Spanned, Value, NU_VARIABLE_ID,
};
use nu_std::load_standard_library;
use nu_utils::{get_default_config, get_default_env};
use std::{
rc::Rc,
sync::{atomic::AtomicBool, Arc},
};
use std::path::{Path, PathBuf};
use std::hint::black_box;
use tango_bench::{benchmark_fn, tango_benchmarks, tango_main, IntoBenchmarks};
fn main() {
// Run registered benchmarks.
divan::main();
}
fn load_bench_commands() -> EngineState {
nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
}
fn canonicalize_path(engine_state: &EngineState, path: &Path) -> PathBuf {
let cwd = engine_state.current_work_dir();
if path.exists() {
match nu_path::canonicalize_with(path, cwd) {
Ok(canon_path) => canon_path,
Err(_) => path.to_owned(),
}
} else {
path.to_owned()
}
}
fn get_home_path(engine_state: &EngineState) -> PathBuf {
nu_path::home_dir()
.map(|path| canonicalize_path(engine_state, &path))
.unwrap_or_default()
}
fn setup_engine() -> EngineState {
let mut engine_state = load_bench_commands();
let cwd = std::env::current_dir()
.unwrap()
.into_os_string()
.into_string()
.unwrap();
let home_path = get_home_path(&engine_state);
// parsing config.nu breaks without PWD set, so set a valid path
engine_state.add_env_var("PWD".into(), Value::string(cwd, Span::test_data()));
engine_state.add_env_var(
"PWD".into(),
Value::string(home_path.to_string_lossy(), Span::test_data()),
);
engine_state.generate_nu_constant();
let nu_const = create_nu_constant(&engine_state, Span::unknown())
.expect("Failed to create nushell constant.");
engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const);
engine_state
}
fn bench_command(bencher: divan::Bencher, scaled_command: String) {
bench_command_with_custom_stack_and_engine(
bencher,
scaled_command,
Stack::new(),
setup_engine(),
)
}
fn bench_command_with_custom_stack_and_engine(
bencher: divan::Bencher,
scaled_command: String,
stack: nu_protocol::engine::Stack,
mut engine: EngineState,
) {
load_standard_library(&mut engine).unwrap();
let commands = Spanned {
span: Span::unknown(),
item: scaled_command,
};
bencher
.with_inputs(|| engine.clone())
.bench_values(|mut engine| {
evaluate_commands(
&commands,
&mut engine,
&mut stack.clone(),
PipelineData::empty(),
None,
false,
)
.unwrap();
})
}
fn setup_stack_and_engine_from_command(command: &str) -> (Stack, EngineState) {
let mut engine = setup_engine();
let commands = Spanned {
@ -45,22 +99,274 @@ fn setup_stack_and_engine_from_command(command: &str) -> (Stack, EngineState) {
};
let mut stack = Stack::new();
// Support running benchmarks with IR mode
stack.use_ir = std::env::var_os("NU_USE_IR").is_some();
evaluate_commands(
&commands,
&mut engine,
&mut stack,
PipelineData::empty(),
Default::default(),
None,
false,
)
.unwrap();
(stack, engine)
}
// FIXME: All benchmarks live in this 1 file to speed up build times when benchmarking.
// When the *_benchmarks functions were in different files, `cargo bench` would build
// an executable for every single one - incredibly slowly. Would be nice to figure out
// a way to split things up again.
#[divan::bench]
fn load_standard_lib(bencher: divan::Bencher) {
let engine = setup_engine();
bencher
.with_inputs(|| engine.clone())
.bench_values(|mut engine| {
load_standard_library(&mut engine).unwrap();
})
}
#[divan::bench_group]
mod record {
use super::*;
fn create_flat_record_string(n: i32) -> String {
let mut s = String::from("let record = {");
for i in 0..n {
s.push_str(&format!("col_{}: {}", i, i));
if i < n - 1 {
s.push_str(", ");
}
}
s.push('}');
s
}
fn create_nested_record_string(depth: i32) -> String {
let mut s = String::from("let record = {");
for _ in 0..depth {
s.push_str("col: {");
}
s.push_str("col_final: 0");
for _ in 0..depth {
s.push('}');
}
s.push('}');
s
}
#[divan::bench(args = [1, 10, 100, 1000])]
fn create(bencher: divan::Bencher, n: i32) {
bench_command(bencher, create_flat_record_string(n));
}
#[divan::bench(args = [1, 10, 100, 1000])]
fn flat_access(bencher: divan::Bencher, n: i32) {
let (stack, engine) = setup_stack_and_engine_from_command(&create_flat_record_string(n));
bench_command_with_custom_stack_and_engine(
bencher,
"$record.col_0 | ignore".to_string(),
stack,
engine,
);
}
#[divan::bench(args = [1, 2, 4, 8, 16, 32, 64, 128])]
fn nest_access(bencher: divan::Bencher, depth: i32) {
let (stack, engine) =
setup_stack_and_engine_from_command(&create_nested_record_string(depth));
let nested_access = ".col".repeat(depth as usize);
bench_command_with_custom_stack_and_engine(
bencher,
format!("$record{} | ignore", nested_access),
stack,
engine,
);
}
}
#[divan::bench_group]
mod table {
use super::*;
fn create_example_table_nrows(n: i32) -> String {
let mut s = String::from("let table = [[foo bar baz]; ");
for i in 0..n {
s.push_str(&format!("[0, 1, {i}]"));
if i < n - 1 {
s.push_str(", ");
}
}
s.push(']');
s
}
#[divan::bench(args = [1, 10, 100, 1000])]
fn create(bencher: divan::Bencher, n: i32) {
bench_command(bencher, create_example_table_nrows(n));
}
#[divan::bench(args = [1, 10, 100, 1000])]
fn get(bencher: divan::Bencher, n: i32) {
let (stack, engine) = setup_stack_and_engine_from_command(&create_example_table_nrows(n));
bench_command_with_custom_stack_and_engine(
bencher,
"$table | get bar | math sum | ignore".to_string(),
stack,
engine,
);
}
#[divan::bench(args = [1, 10, 100, 1000])]
fn select(bencher: divan::Bencher, n: i32) {
let (stack, engine) = setup_stack_and_engine_from_command(&create_example_table_nrows(n));
bench_command_with_custom_stack_and_engine(
bencher,
"$table | select foo baz | ignore".to_string(),
stack,
engine,
);
}
}
#[divan::bench_group]
mod eval_commands {
use super::*;
#[divan::bench(args = [100, 1_000, 10_000])]
fn interleave(bencher: divan::Bencher, n: i32) {
bench_command(
bencher,
format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
)
}
#[divan::bench(args = [100, 1_000, 10_000])]
fn interleave_with_ctrlc(bencher: divan::Bencher, n: i32) {
let mut engine = setup_engine();
engine.ctrlc = Some(std::sync::Arc::new(std::sync::atomic::AtomicBool::new(
false,
)));
load_standard_library(&mut engine).unwrap();
let commands = Spanned {
span: Span::unknown(),
item: format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
};
bencher
.with_inputs(|| engine.clone())
.bench_values(|mut engine| {
evaluate_commands(
&commands,
&mut engine,
&mut nu_protocol::engine::Stack::new(),
PipelineData::empty(),
None,
false,
)
.unwrap();
})
}
#[divan::bench(args = [1, 5, 10, 100, 1_000])]
fn for_range(bencher: divan::Bencher, n: i32) {
bench_command(bencher, format!("(for $x in (1..{}) {{ sleep 50ns }})", n))
}
#[divan::bench(args = [1, 5, 10, 100, 1_000])]
fn each(bencher: divan::Bencher, n: i32) {
bench_command(
bencher,
format!("(1..{}) | each {{|_| sleep 50ns }} | ignore", n),
)
}
#[divan::bench(args = [1, 5, 10, 100, 1_000])]
fn par_each_1t(bencher: divan::Bencher, n: i32) {
bench_command(
bencher,
format!("(1..{}) | par-each -t 1 {{|_| sleep 50ns }} | ignore", n),
)
}
#[divan::bench(args = [1, 5, 10, 100, 1_000])]
fn par_each_2t(bencher: divan::Bencher, n: i32) {
bench_command(
bencher,
format!("(1..{}) | par-each -t 2 {{|_| sleep 50ns }} | ignore", n),
)
}
}
#[divan::bench_group()]
mod parser_benchmarks {
use super::*;
#[divan::bench()]
fn parse_default_config_file(bencher: divan::Bencher) {
let engine_state = setup_engine();
let default_env = get_default_config().as_bytes();
bencher
.with_inputs(|| nu_protocol::engine::StateWorkingSet::new(&engine_state))
.bench_refs(|working_set| parse(working_set, None, default_env, false))
}
#[divan::bench()]
fn parse_default_env_file(bencher: divan::Bencher) {
let engine_state = setup_engine();
let default_env = get_default_env().as_bytes();
bencher
.with_inputs(|| nu_protocol::engine::StateWorkingSet::new(&engine_state))
.bench_refs(|working_set| parse(working_set, None, default_env, false))
}
}
#[divan::bench_group()]
mod eval_benchmarks {
use super::*;
#[divan::bench()]
fn eval_default_env(bencher: divan::Bencher) {
let default_env = get_default_env().as_bytes();
let fname = "default_env.nu";
bencher
.with_inputs(|| (setup_engine(), nu_protocol::engine::Stack::new()))
.bench_values(|(mut engine_state, mut stack)| {
eval_source(
&mut engine_state,
&mut stack,
default_env,
fname,
PipelineData::empty(),
false,
)
})
}
#[divan::bench()]
fn eval_default_config(bencher: divan::Bencher) {
let default_env = get_default_config().as_bytes();
let fname = "default_config.nu";
bencher
.with_inputs(|| (setup_engine(), nu_protocol::engine::Stack::new()))
.bench_values(|(mut engine_state, mut stack)| {
eval_source(
&mut engine_state,
&mut stack,
default_env,
fname,
PipelineData::empty(),
false,
)
})
}
}
// generate a new table data with `row_cnt` rows, `col_cnt` columns.
fn encoding_test_data(row_cnt: usize, col_cnt: usize) -> Value {
let record = Value::test_record(
@ -72,421 +378,76 @@ fn encoding_test_data(row_cnt: usize, col_cnt: usize) -> Value {
Value::list(vec![record; row_cnt], Span::test_data())
}
fn bench_command(
name: &str,
command: &str,
stack: Stack,
engine: EngineState,
) -> impl IntoBenchmarks {
let commands = Spanned {
span: Span::unknown(),
item: command.to_string(),
};
[benchmark_fn(name, move |b| {
let commands = commands.clone();
let stack = stack.clone();
let engine = engine.clone();
b.iter(move || {
let mut stack = stack.clone();
let mut engine = engine.clone();
#[allow(clippy::unit_arg)]
black_box(
evaluate_commands(
&commands,
&mut engine,
&mut stack,
PipelineData::empty(),
Default::default(),
)
.unwrap(),
);
})
})]
}
#[divan::bench_group()]
mod encoding_benchmarks {
use super::*;
fn bench_eval_source(
name: &str,
fname: String,
source: Vec<u8>,
stack: Stack,
engine: EngineState,
) -> impl IntoBenchmarks {
[benchmark_fn(name, move |b| {
let stack = stack.clone();
let engine = engine.clone();
let fname = fname.clone();
let source = source.clone();
b.iter(move || {
let mut stack = stack.clone();
let mut engine = engine.clone();
let fname: &str = &fname.clone();
let source: &[u8] = &source.clone();
black_box(eval_source(
&mut engine,
&mut stack,
source,
fname,
PipelineData::empty(),
false,
));
})
})]
}
/// Load the standard library into the engine.
fn bench_load_standard_lib() -> impl IntoBenchmarks {
[benchmark_fn("load_standard_lib", move |b| {
let engine = setup_engine();
b.iter(move || {
let mut engine = engine.clone();
load_standard_library(&mut engine)
})
})]
}
fn create_flat_record_string(n: i32) -> String {
let mut s = String::from("let record = {");
for i in 0..n {
s.push_str(&format!("col_{}: {}", i, i));
if i < n - 1 {
s.push_str(", ");
}
#[divan::bench(args = [(100, 5), (10000, 15)])]
fn json_encode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
let test_data = PluginOutput::CallResponse(
0,
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
);
let encoder = EncodingType::try_from_bytes(b"json").unwrap();
bencher
.with_inputs(Vec::new)
.bench_values(|mut res| encoder.encode(&test_data, &mut res))
}
s.push('}');
s
}
fn create_nested_record_string(depth: i32) -> String {
let mut s = String::from("let record = {");
for _ in 0..depth {
s.push_str("col: {");
#[divan::bench(args = [(100, 5), (10000, 15)])]
fn msgpack_encode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
let test_data = PluginOutput::CallResponse(
0,
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
);
let encoder = EncodingType::try_from_bytes(b"msgpack").unwrap();
bencher
.with_inputs(Vec::new)
.bench_values(|mut res| encoder.encode(&test_data, &mut res))
}
s.push_str("col_final: 0");
for _ in 0..depth {
s.push('}');
}
s.push('}');
s
}
fn create_example_table_nrows(n: i32) -> String {
let mut s = String::from("let table = [[foo bar baz]; ");
for i in 0..n {
s.push_str(&format!("[0, 1, {i}]"));
if i < n - 1 {
s.push_str(", ");
}
}
s.push(']');
s
}
#[divan::bench_group()]
mod decoding_benchmarks {
use super::*;
fn bench_record_create(n: i32) -> impl IntoBenchmarks {
bench_command(
&format!("record_create_{n}"),
&create_flat_record_string(n),
Stack::new(),
setup_engine(),
)
}
fn bench_record_flat_access(n: i32) -> impl IntoBenchmarks {
let setup_command = create_flat_record_string(n);
let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
bench_command(
&format!("record_flat_access_{n}"),
"$record.col_0 | ignore",
stack,
engine,
)
}
fn bench_record_nested_access(n: i32) -> impl IntoBenchmarks {
let setup_command = create_nested_record_string(n);
let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
let nested_access = ".col".repeat(n as usize);
bench_command(
&format!("record_nested_access_{n}"),
&format!("$record{} | ignore", nested_access),
stack,
engine,
)
}
fn bench_table_create(n: i32) -> impl IntoBenchmarks {
bench_command(
&format!("table_create_{n}"),
&create_example_table_nrows(n),
Stack::new(),
setup_engine(),
)
}
fn bench_table_get(n: i32) -> impl IntoBenchmarks {
let setup_command = create_example_table_nrows(n);
let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
bench_command(
&format!("table_get_{n}"),
"$table | get bar | math sum | ignore",
stack,
engine,
)
}
fn bench_table_select(n: i32) -> impl IntoBenchmarks {
let setup_command = create_example_table_nrows(n);
let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
bench_command(
&format!("table_select_{n}"),
"$table | select foo baz | ignore",
stack,
engine,
)
}
fn bench_eval_interleave(n: i32) -> impl IntoBenchmarks {
let engine = setup_engine();
let stack = Stack::new();
bench_command(
&format!("eval_interleave_{n}"),
&format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
stack,
engine,
)
}
fn bench_eval_interleave_with_interrupt(n: i32) -> impl IntoBenchmarks {
let mut engine = setup_engine();
engine.set_signals(Signals::new(Arc::new(AtomicBool::new(false))));
let stack = Stack::new();
bench_command(
&format!("eval_interleave_with_interrupt_{n}"),
&format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
stack,
engine,
)
}
fn bench_eval_for(n: i32) -> impl IntoBenchmarks {
let engine = setup_engine();
let stack = Stack::new();
bench_command(
&format!("eval_for_{n}"),
&format!("(for $x in (1..{n}) {{ 1 }}) | ignore"),
stack,
engine,
)
}
fn bench_eval_each(n: i32) -> impl IntoBenchmarks {
let engine = setup_engine();
let stack = Stack::new();
bench_command(
&format!("eval_each_{n}"),
&format!("(1..{n}) | each {{|_| 1 }} | ignore"),
stack,
engine,
)
}
fn bench_eval_par_each(n: i32) -> impl IntoBenchmarks {
let engine = setup_engine();
let stack = Stack::new();
bench_command(
&format!("eval_par_each_{n}"),
&format!("(1..{}) | par-each -t 2 {{|_| 1 }} | ignore", n),
stack,
engine,
)
}
fn bench_eval_default_config() -> impl IntoBenchmarks {
let default_env = get_default_config().as_bytes().to_vec();
let fname = "default_config.nu".to_string();
bench_eval_source(
"eval_default_config",
fname,
default_env,
Stack::new(),
setup_engine(),
)
}
fn bench_eval_default_env() -> impl IntoBenchmarks {
let default_env = get_default_env().as_bytes().to_vec();
let fname = "default_env.nu".to_string();
bench_eval_source(
"eval_default_env",
fname,
default_env,
Stack::new(),
setup_engine(),
)
}
fn encode_json(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
let test_data = Rc::new(PluginOutput::CallResponse(
0,
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
));
let encoder = Rc::new(EncodingType::try_from_bytes(b"json").unwrap());
[benchmark_fn(
format!("encode_json_{}_{}", row_cnt, col_cnt),
move |b| {
let encoder = encoder.clone();
let test_data = test_data.clone();
b.iter(move || {
let mut res = Vec::new();
encoder.encode(&*test_data, &mut res).unwrap();
})
},
)]
}
fn encode_msgpack(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
let test_data = Rc::new(PluginOutput::CallResponse(
0,
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
));
let encoder = Rc::new(EncodingType::try_from_bytes(b"msgpack").unwrap());
[benchmark_fn(
format!("encode_msgpack_{}_{}", row_cnt, col_cnt),
move |b| {
let encoder = encoder.clone();
let test_data = test_data.clone();
b.iter(move || {
let mut res = Vec::new();
encoder.encode(&*test_data, &mut res).unwrap();
})
},
)]
}
fn decode_json(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
let test_data = PluginOutput::CallResponse(
0,
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
);
let encoder = EncodingType::try_from_bytes(b"json").unwrap();
let mut res = vec![];
encoder.encode(&test_data, &mut res).unwrap();
[benchmark_fn(
format!("decode_json_{}_{}", row_cnt, col_cnt),
move |b| {
let res = res.clone();
b.iter(move || {
#[divan::bench(args = [(100, 5), (10000, 15)])]
fn json_decode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
let test_data = PluginOutput::CallResponse(
0,
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
);
let encoder = EncodingType::try_from_bytes(b"json").unwrap();
let mut res = vec![];
encoder.encode(&test_data, &mut res).unwrap();
bencher
.with_inputs(|| {
let mut binary_data = std::io::Cursor::new(res.clone());
binary_data.set_position(0);
let _: Result<Option<PluginOutput>, _> =
black_box(encoder.decode(&mut binary_data));
binary_data
})
},
)]
}
.bench_values(|mut binary_data| -> Result<Option<PluginOutput>, _> {
encoder.decode(&mut binary_data)
})
}
fn decode_msgpack(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
let test_data = PluginOutput::CallResponse(
0,
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
);
let encoder = EncodingType::try_from_bytes(b"msgpack").unwrap();
let mut res = vec![];
encoder.encode(&test_data, &mut res).unwrap();
[benchmark_fn(
format!("decode_msgpack_{}_{}", row_cnt, col_cnt),
move |b| {
let res = res.clone();
b.iter(move || {
#[divan::bench(args = [(100, 5), (10000, 15)])]
fn msgpack_decode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
let test_data = PluginOutput::CallResponse(
0,
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
);
let encoder = EncodingType::try_from_bytes(b"msgpack").unwrap();
let mut res = vec![];
encoder.encode(&test_data, &mut res).unwrap();
bencher
.with_inputs(|| {
let mut binary_data = std::io::Cursor::new(res.clone());
binary_data.set_position(0);
let _: Result<Option<PluginOutput>, _> =
black_box(encoder.decode(&mut binary_data));
binary_data
})
},
)]
.bench_values(|mut binary_data| -> Result<Option<PluginOutput>, _> {
encoder.decode(&mut binary_data)
})
}
}
tango_benchmarks!(
bench_load_standard_lib(),
// Data types
// Record
bench_record_create(1),
bench_record_create(10),
bench_record_create(100),
bench_record_create(1_000),
bench_record_flat_access(1),
bench_record_flat_access(10),
bench_record_flat_access(100),
bench_record_flat_access(1_000),
bench_record_nested_access(1),
bench_record_nested_access(2),
bench_record_nested_access(4),
bench_record_nested_access(8),
bench_record_nested_access(16),
bench_record_nested_access(32),
bench_record_nested_access(64),
bench_record_nested_access(128),
// Table
bench_table_create(1),
bench_table_create(10),
bench_table_create(100),
bench_table_create(1_000),
bench_table_get(1),
bench_table_get(10),
bench_table_get(100),
bench_table_get(1_000),
bench_table_select(1),
bench_table_select(10),
bench_table_select(100),
bench_table_select(1_000),
// Eval
// Interleave
bench_eval_interleave(100),
bench_eval_interleave(1_000),
bench_eval_interleave(10_000),
bench_eval_interleave_with_interrupt(100),
bench_eval_interleave_with_interrupt(1_000),
bench_eval_interleave_with_interrupt(10_000),
// For
bench_eval_for(1),
bench_eval_for(10),
bench_eval_for(100),
bench_eval_for(1_000),
bench_eval_for(10_000),
// Each
bench_eval_each(1),
bench_eval_each(10),
bench_eval_each(100),
bench_eval_each(1_000),
bench_eval_each(10_000),
// Par-Each
bench_eval_par_each(1),
bench_eval_par_each(10),
bench_eval_par_each(100),
bench_eval_par_each(1_000),
bench_eval_par_each(10_000),
// Config
bench_eval_default_config(),
// Env
bench_eval_default_env(),
// Encode
// Json
encode_json(100, 5),
encode_json(10000, 15),
// MsgPack
encode_msgpack(100, 5),
encode_msgpack(10000, 15),
// Decode
// Json
decode_json(100, 5),
decode_json(10000, 15),
// MsgPack
decode_msgpack(100, 5),
decode_msgpack(10000, 15)
);
tango_main!();

30
crates/nu-cli/Cargo.toml
View File

@ -5,27 +5,26 @@ repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cli"
edition = "2021"
license = "MIT"
name = "nu-cli"
version = "0.96.2"
version = "0.92.3"
[lib]
bench = false
[dev-dependencies]
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.96.2" }
nu-command = { path = "../nu-command", version = "0.96.2" }
nu-test-support = { path = "../nu-test-support", version = "0.96.2" }
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.92.3" }
nu-command = { path = "../nu-command", version = "0.92.3" }
nu-test-support = { path = "../nu-test-support", version = "0.92.3" }
rstest = { workspace = true, default-features = false }
tempfile = { workspace = true }
[dependencies]
nu-cmd-base = { path = "../nu-cmd-base", version = "0.96.2" }
nu-engine = { path = "../nu-engine", version = "0.96.2" }
nu-path = { path = "../nu-path", version = "0.96.2" }
nu-parser = { path = "../nu-parser", version = "0.96.2" }
nu-plugin-engine = { path = "../nu-plugin-engine", version = "0.96.2", optional = true }
nu-protocol = { path = "../nu-protocol", version = "0.96.2" }
nu-utils = { path = "../nu-utils", version = "0.96.2" }
nu-color-config = { path = "../nu-color-config", version = "0.96.2" }
nu-cmd-base = { path = "../nu-cmd-base", version = "0.92.3" }
nu-engine = { path = "../nu-engine", version = "0.92.3" }
nu-path = { path = "../nu-path", version = "0.92.3" }
nu-parser = { path = "../nu-parser", version = "0.92.3" }
nu-plugin = { path = "../nu-plugin", version = "0.92.3", optional = true }
nu-protocol = { path = "../nu-protocol", version = "0.92.3" }
nu-utils = { path = "../nu-utils", version = "0.92.3" }
nu-color-config = { path = "../nu-color-config", version = "0.92.3" }
nu-ansi-term = { workspace = true }
reedline = { workspace = true, features = ["bashisms", "sqlite"] }
@ -39,11 +38,12 @@ miette = { workspace = true, features = ["fancy-no-backtrace"] }
lscolors = { workspace = true, default-features = false, features = ["nu-ansi-term"] }
once_cell = { workspace = true }
percent-encoding = { workspace = true }
pathdiff = { workspace = true }
sysinfo = { workspace = true }
unicode-segmentation = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
which = { workspace = true }
[features]
plugin = ["nu-plugin-engine"]
system-clipboard = ["reedline/system_clipboard"]
plugin = ["nu-plugin"]
system-clipboard = ["reedline/system_clipboard"]

7
crates/nu-cli/README.md
View File

@ -1,7 +0,0 @@
This crate implements the core functionality of the interactive Nushell REPL and interfaces with `reedline`.
Currently implements the syntax highlighting and completions logic.
Furthermore includes a few commands that are specific to `reedline`
## Internal Nushell crate
This crate implements components of Nushell and is not designed to support plugin authors or other users directly.

70
crates/nu-cli/src/commands/history/history_.rs
View File

@ -47,7 +47,7 @@ impl Command for History {
if let Some(config_path) = nu_path::config_dir() {
let clear = call.has_flag(engine_state, stack, "clear")?;
let long = call.has_flag(engine_state, stack, "long")?;
let signals = engine_state.signals().clone();
let ctrlc = engine_state.ctrlc.clone();
let mut history_path = config_path;
history_path.push("nushell");
@ -67,7 +67,7 @@ impl Command for History {
} else {
let history_reader: Option<Box<dyn ReedlineHistory>> = match history.file_format {
HistoryFileFormat::Sqlite => {
SqliteBackedHistory::with_file(history_path.clone().into(), None, None)
SqliteBackedHistory::with_file(history_path.clone(), None, None)
.map(|inner| {
let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
boxed
@ -77,7 +77,7 @@ impl Command for History {
HistoryFileFormat::PlainText => FileBackedHistory::with_file(
history.max_size as usize,
history_path.clone().into(),
history_path.clone(),
)
.map(|inner| {
let boxed: Box<dyn ReedlineHistory> = Box::new(inner);
@ -107,7 +107,7 @@ impl Command for History {
file: history_path.display().to_string(),
span: head,
})?
.into_pipeline_data(head, signals)),
.into_pipeline_data(ctrlc)),
HistoryFileFormat::Sqlite => Ok(history_reader
.and_then(|h| {
h.search(SearchQuery::everything(SearchDirection::Forward, None))
@ -122,7 +122,7 @@ impl Command for History {
file: history_path.display().to_string(),
span: head,
})?
.into_pipeline_data(head, signals)),
.into_pipeline_data(ctrlc)),
}
}
} else {
@ -156,34 +156,58 @@ fn create_history_record(idx: usize, entry: HistoryItem, long: bool, head: Span)
//2. Create a record of either short or long columns and values
let item_id_value = Value::int(
entry
.id
.and_then(|id| id.to_string().parse::<i64>().ok())
.unwrap_or_default(),
match entry.id {
Some(id) => {
let ids = id.to_string();
match ids.parse::<i64>() {
Ok(i) => i,
_ => 0i64,
}
}
None => 0i64,
},
head,
);
let start_timestamp_value = Value::string(
entry
.start_timestamp
.map(|time| time.to_string())
.unwrap_or_default(),
match entry.start_timestamp {
Some(time) => time.to_string(),
None => "".into(),
},
head,
);
let command_value = Value::string(entry.command_line, head);
let session_id_value = Value::int(
entry
.session_id
.and_then(|id| id.to_string().parse::<i64>().ok())
.unwrap_or_default(),
match entry.session_id {
Some(sid) => {
let sids = sid.to_string();
match sids.parse::<i64>() {
Ok(i) => i,
_ => 0i64,
}
}
None => 0i64,
},
head,
);
let hostname_value = Value::string(
match entry.hostname {
Some(host) => host,
None => "".into(),
},
head,
);
let cwd_value = Value::string(
match entry.cwd {
Some(cwd) => cwd,
None => "".into(),
},
head,
);
let hostname_value = Value::string(entry.hostname.unwrap_or_default(), head);
let cwd_value = Value::string(entry.cwd.unwrap_or_default(), head);
let duration_value = Value::duration(
entry
.duration
.and_then(|d| d.as_nanos().try_into().ok())
.unwrap_or(0),
match entry.duration {
Some(d) => d.as_nanos().try_into().unwrap_or(0),
None => 0,
},
head,
);
let exit_status_value = Value::int(entry.exit_status.unwrap_or(0), head);

12
crates/nu-cli/src/commands/keybindings.rs
View File

@ -36,6 +36,16 @@ For more information on input and keybindings, check:
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
Ok(Value::string(get_full_help(self, engine_state, stack), call.head).into_pipeline_data())
Ok(Value::string(
get_full_help(
&Keybindings.signature(),
&Keybindings.examples(),
engine_state,
stack,
self.is_parser_keyword(),
),
call.head,
)
.into_pipeline_data())
}
}

30
crates/nu-cli/src/commands/keybindings_list.rs
View File

@ -49,26 +49,22 @@ impl Command for KeybindingsList {
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let all_options = ["modifiers", "keycodes", "edits", "modes", "events"];
let presence = all_options
.iter()
.map(|option| call.has_flag(engine_state, stack, option))
.collect::<Result<Vec<_>, ShellError>>()?;
let no_option_specified = presence.iter().all(|present| !*present);
let records = all_options
.iter()
.zip(presence)
.filter(|(_, present)| no_option_specified || *present)
.flat_map(|(option, _)| get_records(option, call.head))
.collect();
let records = if call.named_len() == 0 {
let all_options = ["modifiers", "keycodes", "edits", "modes", "events"];
all_options
.iter()
.flat_map(|argument| get_records(argument, call.head))
.collect()
} else {
call.named_iter()
.flat_map(|(argument, _, _)| get_records(argument.item.as_str(), call.head))
.collect()
};
Ok(Value::list(records, call.head).into_pipeline_data())
}

38
crates/nu-cli/src/completions/base.rs
View File

@ -1,23 +1,45 @@
use crate::completions::CompletionOptions;
use nu_protocol::{
engine::{Stack, StateWorkingSet},
Span,
};
use crate::completions::{CompletionOptions, SortBy};
use nu_protocol::{engine::StateWorkingSet, levenshtein_distance, Span};
use reedline::Suggestion;
// Completer trait represents the three stages of the completion
// fetch, filter and sort
pub trait Completer {
/// Fetch, filter, and sort completions
#[allow(clippy::too_many_arguments)]
fn fetch(
&mut self,
working_set: &StateWorkingSet,
stack: &Stack,
prefix: Vec<u8>,
span: Span,
offset: usize,
pos: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion>;
fn get_sort_by(&self) -> SortBy {
SortBy::Ascending
}
fn sort(&self, items: Vec<SemanticSuggestion>, prefix: Vec<u8>) -> Vec<SemanticSuggestion> {
let prefix_str = String::from_utf8_lossy(&prefix).to_string();
let mut filtered_items = items;
// Sort items
match self.get_sort_by() {
SortBy::LevenshteinDistance => {
filtered_items.sort_by(|a, b| {
let a_distance = levenshtein_distance(&prefix_str, &a.suggestion.value);
let b_distance = levenshtein_distance(&prefix_str, &b.suggestion.value);
a_distance.cmp(&b_distance)
});
}
SortBy::Ascending => {
filtered_items.sort_by(|a, b| a.suggestion.value.cmp(&b.suggestion.value));
}
SortBy::None => {}
};
filtered_items
}
}
#[derive(Debug, Default, PartialEq)]

45
crates/nu-cli/src/completions/command_completions.rs
View File

@ -1,17 +1,19 @@
use crate::{
completions::{Completer, CompletionOptions, MatchAlgorithm},
completions::{Completer, CompletionOptions, MatchAlgorithm, SortBy},
SuggestionKind,
};
use nu_parser::FlatShape;
use nu_protocol::{
engine::{CachedFile, Stack, StateWorkingSet},
engine::{CachedFile, EngineState, StateWorkingSet},
Span,
};
use reedline::Suggestion;
use std::sync::Arc;
use super::{completion_common::sort_suggestions, SemanticSuggestion};
use super::SemanticSuggestion;
pub struct CommandCompletion {
engine_state: Arc<EngineState>,
flattened: Vec<(Span, FlatShape)>,
flat_shape: FlatShape,
force_completion_after_space: bool,
@ -19,11 +21,14 @@ pub struct CommandCompletion {
impl CommandCompletion {
pub fn new(
engine_state: Arc<EngineState>,
_: &StateWorkingSet,
flattened: Vec<(Span, FlatShape)>,
flat_shape: FlatShape,
force_completion_after_space: bool,
) -> Self {
Self {
engine_state,
flattened,
flat_shape,
force_completion_after_space,
@ -32,14 +37,13 @@ impl CommandCompletion {
fn external_command_completion(
&self,
working_set: &StateWorkingSet,
prefix: &str,
match_algorithm: MatchAlgorithm,
) -> Vec<String> {
let mut executables = vec![];
// os agnostic way to get the PATH env var
let paths = working_set.permanent_state.get_path_env_var();
let paths = self.engine_state.get_path_env_var();
if let Some(paths) = paths {
if let Ok(paths) = paths.as_list() {
@ -48,10 +52,7 @@ impl CommandCompletion {
if let Ok(mut contents) = std::fs::read_dir(path.as_ref()) {
while let Some(Ok(item)) = contents.next() {
if working_set
.permanent_state
.config
.max_external_completion_results
if self.engine_state.config.max_external_completion_results
> executables.len() as i64
&& !executables.contains(
&item
@ -99,9 +100,10 @@ impl CommandCompletion {
suggestion: Suggestion {
value: String::from_utf8_lossy(&x.0).to_string(),
description: x.1,
style: None,
extra: None,
span: reedline::Span::new(span.start - offset, span.end - offset),
append_whitespace: true,
..Suggestion::default()
},
kind: Some(SuggestionKind::Command(x.2)),
})
@ -112,14 +114,16 @@ impl CommandCompletion {
if find_externals {
let results_external = self
.external_command_completion(working_set, &partial, match_algorithm)
.external_command_completion(&partial, match_algorithm)
.into_iter()
.map(move |x| SemanticSuggestion {
suggestion: Suggestion {
value: x,
description: None,
style: None,
extra: None,
span: reedline::Span::new(span.start - offset, span.end - offset),
append_whitespace: true,
..Suggestion::default()
},
// TODO: is there a way to create a test?
kind: None,
@ -133,9 +137,11 @@ impl CommandCompletion {
results.push(SemanticSuggestion {
suggestion: Suggestion {
value: format!("^{}", external.suggestion.value),
description: None,
style: None,
extra: None,
span: external.suggestion.span,
append_whitespace: true,
..Suggestion::default()
},
kind: external.kind,
})
@ -155,8 +161,7 @@ impl Completer for CommandCompletion {
fn fetch(
&mut self,
working_set: &StateWorkingSet,
_stack: &Stack,
prefix: Vec<u8>,
_prefix: Vec<u8>,
span: Span,
offset: usize,
pos: usize,
@ -193,7 +198,7 @@ impl Completer for CommandCompletion {
};
if !subcommands.is_empty() {
return sort_suggestions(&String::from_utf8_lossy(&prefix), subcommands, options);
return subcommands;
}
let config = working_set.get_config();
@ -218,7 +223,11 @@ impl Completer for CommandCompletion {
vec![]
};
sort_suggestions(&String::from_utf8_lossy(&prefix), commands, options)
subcommands.into_iter().chain(commands).collect::<Vec<_>>()
}
fn get_sort_by(&self) -> SortBy {
SortBy::LevenshteinDistance
}
}
@ -257,8 +266,6 @@ pub fn is_passthrough_command(working_set_file_contents: &[CachedFile]) -> bool
#[cfg(test)]
mod command_completions_tests {
use super::*;
use nu_protocol::engine::EngineState;
use std::sync::Arc;
#[test]
fn test_find_non_whitespace_index() {

76
crates/nu-cli/src/completions/completer.rs
View File

@ -22,10 +22,10 @@ pub struct NuCompleter {
}
impl NuCompleter {
pub fn new(engine_state: Arc<EngineState>, stack: Arc<Stack>) -> Self {
pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
Self {
engine_state,
stack: Stack::with_parent(stack).reset_out_dest().capture(),
stack: stack.reset_out_dest().capture(),
}
}
@ -48,19 +48,17 @@ impl NuCompleter {
let options = CompletionOptions {
case_sensitive: config.case_sensitive_completions,
match_algorithm: config.completion_algorithm.into(),
sort: config.completion_sort,
..Default::default()
};
completer.fetch(
working_set,
&self.stack,
prefix.clone(),
new_span,
offset,
pos,
&options,
)
// Fetch
let mut suggestions =
completer.fetch(working_set, prefix.clone(), new_span, offset, pos, &options);
// Sort
suggestions = completer.sort(suggestions, prefix);
suggestions
}
fn external_completion(
@ -98,8 +96,9 @@ impl NuCompleter {
PipelineData::empty(),
);
match result.and_then(|data| data.into_value(span)) {
Ok(value) => {
match result {
Ok(pd) => {
let value = pd.into_value(span);
if let Value::List { vals, .. } = value {
let result =
map_value_completions(vals.iter(), Span::new(span.start, span.end), offset);
@ -176,8 +175,11 @@ impl NuCompleter {
// Variables completion
if prefix.starts_with(b"$") || most_left_var.is_some() {
let mut completer =
VariableCompletion::new(most_left_var.unwrap_or((vec![], vec![])));
let mut completer = VariableCompletion::new(
self.engine_state.clone(),
self.stack.clone(),
most_left_var.unwrap_or((vec![], vec![])),
);
return self.process_completion(
&mut completer,
@ -222,6 +224,8 @@ impl NuCompleter {
|| (flat_idx == 0 && working_set.get_span_contents(new_span).is_empty())
{
let mut completer = CommandCompletion::new(
self.engine_state.clone(),
&working_set,
flattened.clone(),
// flat_idx,
FlatShape::String,
@ -249,7 +253,10 @@ impl NuCompleter {
|| prev_expr_str == b"overlay use"
|| prev_expr_str == b"source-env"
{
let mut completer = DotNuCompletion::new();
let mut completer = DotNuCompletion::new(
self.engine_state.clone(),
self.stack.clone(),
);
return self.process_completion(
&mut completer,
@ -260,7 +267,10 @@ impl NuCompleter {
pos,
);
} else if prev_expr_str == b"ls" {
let mut completer = FileCompletion::new();
let mut completer = FileCompletion::new(
self.engine_state.clone(),
self.stack.clone(),
);
return self.process_completion(
&mut completer,
@ -278,6 +288,7 @@ impl NuCompleter {
match &flat.1 {
FlatShape::Custom(decl_id) => {
let mut completer = CustomCompletion::new(
self.engine_state.clone(),
self.stack.clone(),
*decl_id,
initial_line,
@ -293,7 +304,10 @@ impl NuCompleter {
);
}
FlatShape::Directory => {
let mut completer = DirectoryCompletion::new();
let mut completer = DirectoryCompletion::new(
self.engine_state.clone(),
self.stack.clone(),
);
return self.process_completion(
&mut completer,
@ -305,7 +319,10 @@ impl NuCompleter {
);
}
FlatShape::Filepath | FlatShape::GlobPattern => {
let mut completer = FileCompletion::new();
let mut completer = FileCompletion::new(
self.engine_state.clone(),
self.stack.clone(),
);
return self.process_completion(
&mut completer,
@ -318,6 +335,8 @@ impl NuCompleter {
}
flat_shape => {
let mut completer = CommandCompletion::new(
self.engine_state.clone(),
&working_set,
flattened.clone(),
// flat_idx,
flat_shape.clone(),
@ -350,7 +369,10 @@ impl NuCompleter {
}
// Check for file completion
let mut completer = FileCompletion::new();
let mut completer = FileCompletion::new(
self.engine_state.clone(),
self.stack.clone(),
);
out = self.process_completion(
&mut completer,
&working_set,
@ -444,11 +466,14 @@ pub fn map_value_completions<'a>(
return Some(SemanticSuggestion {
suggestion: Suggestion {
value: s,
description: None,
style: None,
extra: None,
span: reedline::Span {
start: span.start - offset,
end: span.end - offset,
},
..Suggestion::default()
append_whitespace: false,
},
kind: Some(SuggestionKind::Type(x.get_type())),
});
@ -458,11 +483,14 @@ pub fn map_value_completions<'a>(
if let Ok(record) = x.as_record() {
let mut suggestion = Suggestion {
value: String::from(""), // Initialize with empty string
description: None,
style: None,
extra: None,
span: reedline::Span {
start: span.start - offset,
end: span.end - offset,
},
..Suggestion::default()
append_whitespace: false,
};
// Iterate the cols looking for `value` and `description`
@ -529,7 +557,7 @@ mod completer_tests {
result.err().unwrap()
);
let mut completer = NuCompleter::new(engine_state.into(), Arc::new(Stack::new()));
let mut completer = NuCompleter::new(engine_state.into(), Stack::new());
let dataset = [
("sudo", false, "", Vec::new()),
("sudo l", true, "l", vec!["ls", "let", "lines", "loop"]),

302
crates/nu-cli/src/completions/completion_common.rs
View File

@ -1,133 +1,81 @@
use crate::{
completions::{matches, CompletionOptions},
SemanticSuggestion,
};
use fuzzy_matcher::{skim::SkimMatcherV2, FuzzyMatcher};
use crate::completions::{matches, CompletionOptions};
use nu_ansi_term::Style;
use nu_engine::env_to_string;
use nu_path::{expand_to_real_path, home_dir};
use nu_path::home_dir;
use nu_protocol::{
engine::{EngineState, Stack, StateWorkingSet},
CompletionSort, Span,
Span,
};
use nu_utils::get_ls_colors;
use std::path::{is_separator, Component, Path, PathBuf, MAIN_SEPARATOR as SEP};
use std::{
ffi::OsStr,
path::{is_separator, Component, Path, PathBuf, MAIN_SEPARATOR as SEP},
};
use super::MatchAlgorithm;
#[derive(Clone, Default)]
pub struct PathBuiltFromString {
parts: Vec<String>,
isdir: bool,
}
/// Recursively goes through paths that match a given `partial`.
/// built: State struct for a valid matching path built so far.
///
/// `isdir`: whether the current partial path has a trailing slash.
/// Parsing a path string into a pathbuf loses that bit of information.
///
/// want_directory: Whether we want only directories as completion matches.
/// Some commands like `cd` can only be run on directories whereas others
/// like `ls` can be run on regular files as well.
pub fn complete_rec(
partial: &[&str],
built: &PathBuiltFromString,
fn complete_rec(
partial: &[String],
cwd: &Path,
options: &CompletionOptions,
want_directory: bool,
dir: bool,
isdir: bool,
) -> Vec<PathBuiltFromString> {
) -> Vec<PathBuf> {
let mut completions = vec![];
if let Some((&base, rest)) = partial.split_first() {
if (base == "." || base == "..") && (isdir || !rest.is_empty()) {
let mut built = built.clone();
built.parts.push(base.to_string());
built.isdir = true;
return complete_rec(rest, &built, cwd, options, want_directory, isdir);
}
}
if let Ok(result) = cwd.read_dir() {
for entry in result.filter_map(|e| e.ok()) {
let entry_name = entry.file_name().to_string_lossy().into_owned();
let path = entry.path();
let mut built_path = cwd.to_path_buf();
for part in &built.parts {
built_path.push(part);
}
let Ok(result) = built_path.read_dir() else {
return completions;
};
let mut entries = Vec::new();
for entry in result.filter_map(|e| e.ok()) {
let entry_name = entry.file_name().to_string_lossy().into_owned();
let entry_isdir = entry.path().is_dir();
let mut built = built.clone();
built.parts.push(entry_name.clone());
built.isdir = entry_isdir;
if !want_directory || entry_isdir {
entries.push((entry_name, built));
}
}
let prefix = partial.first().unwrap_or(&"");
let sorted_entries = sort_completions(prefix, entries, options, |(entry, _)| entry);
for (entry_name, built) in sorted_entries {
match partial.split_first() {
Some((base, rest)) => {
if matches(base, &entry_name, options) {
// We use `isdir` to confirm that the current component has
// at least one next component or a slash.
// Serves as confirmation to ignore longer completions for
// components in between.
if !rest.is_empty() || isdir {
completions.extend(complete_rec(
rest,
&built,
cwd,
options,
want_directory,
isdir,
));
} else {
completions.push(built);
if !dir || path.is_dir() {
match partial.first() {
Some(base) if matches(base, &entry_name, options) => {
let partial = &partial[1..];
if !partial.is_empty() || isdir {
completions.extend(complete_rec(partial, &path, options, dir, isdir));
if entry_name.eq(base) {
break;
}
} else {
completions.push(path)
}
}
None => completions.push(path),
_ => {}
}
if entry_name.eq(base)
&& matches!(options.match_algorithm, MatchAlgorithm::Prefix)
&& isdir
{
break;
}
}
None => {
completions.push(built);
}
}
}
completions
}
#[derive(Debug)]
enum OriginalCwd {
None,
Home,
Prefix(String),
Home(PathBuf),
Some(PathBuf),
// referencing a single local file
Local(PathBuf),
}
impl OriginalCwd {
fn apply(&self, mut p: PathBuiltFromString, path_separator: char) -> String {
match self {
Self::None => {}
Self::Home => p.parts.insert(0, "~".to_string()),
Self::Prefix(s) => p.parts.insert(0, s.clone()),
fn apply(&self, p: &Path) -> String {
let mut ret = match self {
Self::None => p.to_string_lossy().into_owned(),
Self::Some(base) => pathdiff::diff_paths(p, base)
.unwrap_or(p.to_path_buf())
.to_string_lossy()
.into_owned(),
Self::Home(home) => match p.strip_prefix(home) {
Ok(suffix) => format!("~{}{}", SEP, suffix.to_string_lossy()),
_ => p.to_string_lossy().into_owned(),
},
Self::Local(base) => Path::new(".")
.join(pathdiff::diff_paths(p, base).unwrap_or(p.to_path_buf()))
.to_string_lossy()
.into_owned(),
};
let mut ret = p.parts.join(&path_separator.to_string());
if p.isdir {
ret.push(path_separator);
if p.is_dir() {
ret.push(SEP);
}
ret
}
@ -158,14 +106,6 @@ pub fn complete_item(
) -> Vec<(nu_protocol::Span, String, Option<Style>)> {
let partial = surround_remove(partial);
let isdir = partial.ends_with(is_separator);
#[cfg(unix)]
let path_separator = SEP;
#[cfg(windows)]
let path_separator = partial
.chars()
.rfind(|c: &char| is_separator(*c))
.unwrap_or(SEP);
let cwd_pathbuf = Path::new(cwd).to_path_buf();
let ls_colors = (engine_state.config.use_ls_colors_completions
&& engine_state.config.use_ansi_coloring)
@ -176,72 +116,79 @@ pub fn complete_item(
};
get_ls_colors(ls_colors_env_str)
});
let mut cwd = cwd_pathbuf.clone();
let mut prefix_len = 0;
let mut original_cwd = OriginalCwd::None;
let mut components_vec: Vec<Component> = Path::new(&partial).components().collect();
let mut components = Path::new(&partial).components().peekable();
match components.peek().cloned() {
// Path components that end with a single "." get normalized away,
// so if the partial path ends in a literal "." we must add it back in manually
if partial.ends_with('.') && partial.len() > 1 {
components_vec.push(Component::Normal(OsStr::new(".")));
};
let mut components = components_vec.into_iter().peekable();
let mut cwd = match components.peek().cloned() {
Some(c @ Component::Prefix(..)) => {
// windows only by definition
components.next();
if let Some(Component::RootDir) = components.peek().cloned() {
components.next();
};
cwd = [c, Component::RootDir].iter().collect();
prefix_len = c.as_os_str().len();
original_cwd = OriginalCwd::Prefix(c.as_os_str().to_string_lossy().into_owned());
[c, Component::RootDir].iter().collect()
}
Some(c @ Component::RootDir) => {
components.next();
// This is kind of a hack. When joining an empty string with the rest,
// we add the slash automagically
cwd = PathBuf::from(c.as_os_str());
prefix_len = 1;
original_cwd = OriginalCwd::Prefix(String::new());
PathBuf::from(c.as_os_str())
}
Some(Component::Normal(home)) if home.to_string_lossy() == "~" => {
components.next();
cwd = home_dir().map(Into::into).unwrap_or(cwd_pathbuf);
prefix_len = 1;
original_cwd = OriginalCwd::Home;
original_cwd = OriginalCwd::Home(home_dir().unwrap_or(cwd_pathbuf.clone()));
home_dir().unwrap_or(cwd_pathbuf)
}
Some(Component::CurDir) => {
components.next();
original_cwd = match components.peek().cloned() {
Some(Component::Normal(_)) | None => OriginalCwd::Local(cwd_pathbuf.clone()),
_ => OriginalCwd::Some(cwd_pathbuf.clone()),
};
cwd_pathbuf
}
_ => {
original_cwd = OriginalCwd::Some(cwd_pathbuf.clone());
cwd_pathbuf
}
_ => {}
};
let after_prefix = &partial[prefix_len..];
let partial: Vec<_> = after_prefix
.strip_prefix(is_separator)
.unwrap_or(after_prefix)
.split(is_separator)
.filter(|s| !s.is_empty())
.collect();
let mut partial = vec![];
complete_rec(
partial.as_slice(),
&PathBuiltFromString::default(),
&cwd,
options,
want_directory,
isdir,
)
.into_iter()
.map(|p| {
let path = original_cwd.apply(p, path_separator);
let style = ls_colors.as_ref().map(|lsc| {
lsc.style_for_path_with_metadata(
&path,
std::fs::symlink_metadata(expand_to_real_path(&path))
.ok()
.as_ref(),
)
.map(lscolors::Style::to_nu_ansi_term_style)
.unwrap_or_default()
});
(span, escape_path(path, want_directory), style)
})
.collect()
for component in components {
match component {
Component::Prefix(..) => unreachable!(),
Component::RootDir => unreachable!(),
Component::CurDir => {}
Component::ParentDir => {
if partial.pop().is_none() {
cwd.pop();
}
}
Component::Normal(c) => partial.push(c.to_string_lossy().into_owned()),
}
}
complete_rec(partial.as_slice(), &cwd, options, want_directory, isdir)
.into_iter()
.map(|p| {
let path = original_cwd.apply(&p);
let style = ls_colors.as_ref().map(|lsc| {
lsc.style_for_path_with_metadata(
&path,
std::fs::symlink_metadata(&path).ok().as_ref(),
)
.map(lscolors::Style::to_nu_ansi_term_style)
.unwrap_or_default()
});
(span, escape_path(path, want_directory), style)
})
.collect()
}
// Fix files or folders with quotes or hashes
@ -301,42 +248,3 @@ pub fn adjust_if_intermediate(
readjusted,
}
}
/// Convenience function to sort suggestions using [`sort_completions`]
pub fn sort_suggestions(
prefix: &str,
items: Vec<SemanticSuggestion>,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
sort_completions(prefix, items, options, |it| &it.suggestion.value)
}
/// # Arguments
/// * `prefix` - What the user's typed, for sorting by fuzzy matcher score
pub fn sort_completions<T>(
prefix: &str,
mut items: Vec<T>,
options: &CompletionOptions,
get_value: fn(&T) -> &str,
) -> Vec<T> {
// Sort items
if options.sort == CompletionSort::Smart && options.match_algorithm == MatchAlgorithm::Fuzzy {
let mut matcher = SkimMatcherV2::default();
if options.case_sensitive {
matcher = matcher.respect_case();
} else {
matcher = matcher.ignore_case();
};
items.sort_by(|a, b| {
let a_str = get_value(a);
let b_str = get_value(b);
let a_score = matcher.fuzzy_match(a_str, prefix).unwrap_or_default();
let b_score = matcher.fuzzy_match(b_str, prefix).unwrap_or_default();
b_score.cmp(&a_score).then(a_str.cmp(b_str))
});
} else {
items.sort_by(|a, b| get_value(a).cmp(get_value(b)));
}
items
}

13
crates/nu-cli/src/completions/completion_options.rs
View File

@ -1,10 +1,17 @@
use fuzzy_matcher::{skim::SkimMatcherV2, FuzzyMatcher};
use nu_parser::trim_quotes_str;
use nu_protocol::{CompletionAlgorithm, CompletionSort};
use nu_protocol::CompletionAlgorithm;
use std::fmt::Display;
#[derive(Copy, Clone)]
pub enum SortBy {
LevenshteinDistance,
Ascending,
None,
}
/// Describes how suggestions should be matched.
#[derive(Copy, Clone, Debug, PartialEq)]
#[derive(Copy, Clone, Debug)]
pub enum MatchAlgorithm {
/// Only show suggestions which begin with the given input
///
@ -89,7 +96,6 @@ pub struct CompletionOptions {
pub case_sensitive: bool,
pub positional: bool,
pub match_algorithm: MatchAlgorithm,
pub sort: CompletionSort,
}
impl Default for CompletionOptions {
@ -98,7 +104,6 @@ impl Default for CompletionOptions {
case_sensitive: true,
positional: true,
match_algorithm: MatchAlgorithm::Prefix,
sort: Default::default(),
}
}
}

150
crates/nu-cli/src/completions/custom_completions.rs
View File

@ -1,31 +1,33 @@
use crate::completions::{
completer::map_value_completions, Completer, CompletionOptions, MatchAlgorithm,
SemanticSuggestion,
SemanticSuggestion, SortBy,
};
use nu_engine::eval_call;
use nu_protocol::{
ast::{Argument, Call, Expr, Expression},
debugger::WithoutDebug,
engine::{Stack, StateWorkingSet},
CompletionSort, PipelineData, Span, Type, Value,
engine::{EngineState, Stack, StateWorkingSet},
PipelineData, Span, Type, Value,
};
use nu_utils::IgnoreCaseExt;
use std::collections::HashMap;
use super::completion_common::sort_suggestions;
use std::{collections::HashMap, sync::Arc};
pub struct CustomCompletion {
engine_state: Arc<EngineState>,
stack: Stack,
decl_id: usize,
line: String,
sort_by: SortBy,
}
impl CustomCompletion {
pub fn new(stack: Stack, decl_id: usize, line: String) -> Self {
pub fn new(engine_state: Arc<EngineState>, stack: Stack, decl_id: usize, line: String) -> Self {
Self {
stack,
engine_state,
stack: stack.reset_out_dest().capture(),
decl_id,
line,
sort_by: SortBy::None,
}
}
}
@ -33,8 +35,7 @@ impl CustomCompletion {
impl Completer for CustomCompletion {
fn fetch(
&mut self,
working_set: &StateWorkingSet,
_stack: &Stack,
_: &StateWorkingSet,
prefix: Vec<u8>,
span: Span,
offset: usize,
@ -46,22 +47,24 @@ impl Completer for CustomCompletion {
// Call custom declaration
let result = eval_call::<WithoutDebug>(
working_set.permanent_state,
&self.engine_state,
&mut self.stack,
&Call {
decl_id: self.decl_id,
head: span,
arguments: vec![
Argument::Positional(Expression::new_unknown(
Expr::String(self.line.clone()),
Span::unknown(),
Type::String,
)),
Argument::Positional(Expression::new_unknown(
Expr::Int(line_pos as i64),
Span::unknown(),
Type::Int,
)),
Argument::Positional(Expression {
span: Span::unknown(),
ty: Type::String,
expr: Expr::String(self.line.clone()),
custom_completion: None,
}),
Argument::Positional(Expression {
span: Span::unknown(),
ty: Type::Int,
expr: Expr::Int(line_pos as i64),
custom_completion: None,
}),
],
parser_info: HashMap::new(),
},
@ -72,62 +75,67 @@ impl Completer for CustomCompletion {
// Parse result
let suggestions = result
.and_then(|data| data.into_value(span))
.map(|value| match &value {
Value::Record { val, .. } => {
let completions = val
.get("completions")
.and_then(|val| {
val.as_list()
.ok()
.map(|it| map_value_completions(it.iter(), span, offset))
})
.unwrap_or_default();
let options = val.get("options");
if let Some(Value::Record { val: options, .. }) = &options {
let should_sort = options
.get("sort")
.and_then(|val| val.as_bool().ok())
.unwrap_or(false);
custom_completion_options = Some(CompletionOptions {
case_sensitive: options
.get("case_sensitive")
.and_then(|val| val.as_bool().ok())
.unwrap_or(true),
positional: options
.get("positional")
.and_then(|val| val.as_bool().ok())
.unwrap_or(true),
match_algorithm: match options.get("completion_algorithm") {
Some(option) => option
.coerce_string()
.map(|pd| {
let value = pd.into_value(span);
match &value {
Value::Record { val, .. } => {
let completions = val
.get("completions")
.and_then(|val| {
val.as_list()
.ok()
.and_then(|option| option.try_into().ok())
.unwrap_or(MatchAlgorithm::Prefix),
None => completion_options.match_algorithm,
},
sort: if should_sort {
CompletionSort::Alphabetical
} else {
CompletionSort::Smart
},
});
}
.map(|it| map_value_completions(it.iter(), span, offset))
})
.unwrap_or_default();
let options = val.get("options");
completions
if let Some(Value::Record { val: options, .. }) = &options {
let should_sort = options
.get("sort")
.and_then(|val| val.as_bool().ok())
.unwrap_or(false);
if should_sort {
self.sort_by = SortBy::Ascending;
}
custom_completion_options = Some(CompletionOptions {
case_sensitive: options
.get("case_sensitive")
.and_then(|val| val.as_bool().ok())
.unwrap_or(true),
positional: options
.get("positional")
.and_then(|val| val.as_bool().ok())
.unwrap_or(true),
match_algorithm: match options.get("completion_algorithm") {
Some(option) => option
.coerce_string()
.ok()
.and_then(|option| option.try_into().ok())
.unwrap_or(MatchAlgorithm::Prefix),
None => completion_options.match_algorithm,
},
});
}
completions
}
Value::List { vals, .. } => map_value_completions(vals.iter(), span, offset),
_ => vec![],
}
Value::List { vals, .. } => map_value_completions(vals.iter(), span, offset),
_ => vec![],
})
.unwrap_or_default();
let options = custom_completion_options
.as_ref()
.unwrap_or(completion_options);
let suggestions = filter(&prefix, suggestions, completion_options);
sort_suggestions(&String::from_utf8_lossy(&prefix), suggestions, options)
if let Some(custom_completion_options) = custom_completion_options {
filter(&prefix, suggestions, &custom_completion_options)
} else {
filter(&prefix, suggestions, completion_options)
}
}
fn get_sort_by(&self) -> SortBy {
self.sort_by
}
}

71
crates/nu-cli/src/completions/directory_completions.rs
View File

@ -1,23 +1,32 @@
use crate::completions::{
completion_common::{adjust_if_intermediate, complete_item, AdjustView},
Completer, CompletionOptions,
Completer, CompletionOptions, SortBy,
};
use nu_ansi_term::Style;
use nu_protocol::{
engine::{EngineState, Stack, StateWorkingSet},
Span,
levenshtein_distance, Span,
};
use reedline::Suggestion;
use std::path::Path;
use std::{
path::{Path, MAIN_SEPARATOR as SEP},
sync::Arc,
};
use super::SemanticSuggestion;
#[derive(Clone, Default)]
pub struct DirectoryCompletion {}
#[derive(Clone)]
pub struct DirectoryCompletion {
engine_state: Arc<EngineState>,
stack: Stack,
}
impl DirectoryCompletion {
pub fn new() -> Self {
Self::default()
pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
Self {
engine_state,
stack,
}
}
}
@ -25,46 +34,76 @@ impl Completer for DirectoryCompletion {
fn fetch(
&mut self,
working_set: &StateWorkingSet,
stack: &Stack,
prefix: Vec<u8>,
span: Span,
offset: usize,
_pos: usize,
_: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
let AdjustView { prefix, span, .. } = adjust_if_intermediate(&prefix, working_set, span);
// Filter only the folders
#[allow(deprecated)]
let items: Vec<_> = directory_completion(
let output: Vec<_> = directory_completion(
span,
&prefix,
&working_set.permanent_state.current_work_dir(),
&self.engine_state.current_work_dir(),
options,
working_set.permanent_state,
stack,
self.engine_state.as_ref(),
&self.stack,
)
.into_iter()
.map(move |x| SemanticSuggestion {
suggestion: Suggestion {
value: x.1,
description: None,
style: x.2,
extra: None,
span: reedline::Span {
start: x.0.start - offset,
end: x.0.end - offset,
},
..Suggestion::default()
append_whitespace: false,
},
// TODO????
kind: None,
})
.collect();
output
}
// Sort results prioritizing the non hidden folders
fn sort(&self, items: Vec<SemanticSuggestion>, prefix: Vec<u8>) -> Vec<SemanticSuggestion> {
let prefix_str = String::from_utf8_lossy(&prefix).to_string();
// Sort items
let mut sorted_items = items;
match self.get_sort_by() {
SortBy::Ascending => {
sorted_items.sort_by(|a, b| {
// Ignore trailing slashes in folder names when sorting
a.suggestion
.value
.trim_end_matches(SEP)
.cmp(b.suggestion.value.trim_end_matches(SEP))
});
}
SortBy::LevenshteinDistance => {
sorted_items.sort_by(|a, b| {
let a_distance = levenshtein_distance(&prefix_str, &a.suggestion.value);
let b_distance = levenshtein_distance(&prefix_str, &b.suggestion.value);
a_distance.cmp(&b_distance)
});
}
_ => (),
}
// Separate the results between hidden and non hidden
let mut hidden: Vec<SemanticSuggestion> = vec![];
let mut non_hidden: Vec<SemanticSuggestion> = vec![];
for item in items.into_iter() {
for item in sorted_items.into_iter() {
let item_path = Path::new(&item.suggestion.value);
if let Some(value) = item_path.file_name() {

83
crates/nu-cli/src/completions/dotnu_completions.rs
View File

@ -1,31 +1,39 @@
use crate::completions::{file_path_completion, Completer, CompletionOptions};
use crate::completions::{file_path_completion, Completer, CompletionOptions, SortBy};
use nu_protocol::{
engine::{Stack, StateWorkingSet},
engine::{EngineState, Stack, StateWorkingSet},
Span,
};
use reedline::Suggestion;
use std::path::{is_separator, Path, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR};
use std::{
path::{is_separator, Path, MAIN_SEPARATOR as SEP, MAIN_SEPARATOR_STR},
sync::Arc,
};
use super::{completion_common::sort_suggestions, SemanticSuggestion};
use super::SemanticSuggestion;
#[derive(Clone, Default)]
pub struct DotNuCompletion {}
#[derive(Clone)]
pub struct DotNuCompletion {
engine_state: Arc<EngineState>,
stack: Stack,
}
impl DotNuCompletion {
pub fn new() -> Self {
Self::default()
pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
Self {
engine_state,
stack,
}
}
}
impl Completer for DotNuCompletion {
fn fetch(
&mut self,
working_set: &StateWorkingSet,
stack: &Stack,
_: &StateWorkingSet,
prefix: Vec<u8>,
span: Span,
offset: usize,
_pos: usize,
_: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
let prefix_str = String::from_utf8_lossy(&prefix).replace('`', "");
@ -41,25 +49,26 @@ impl Completer for DotNuCompletion {
let mut is_current_folder = false;
// Fetch the lib dirs
let lib_dirs: Vec<String> = if let Some(lib_dirs) = working_set.get_env_var("NU_LIB_DIRS") {
lib_dirs
.as_list()
.into_iter()
.flat_map(|it| {
it.iter().map(|x| {
x.to_path()
.expect("internal error: failed to convert lib path")
let lib_dirs: Vec<String> =
if let Some(lib_dirs) = self.engine_state.get_env_var("NU_LIB_DIRS") {
lib_dirs
.as_list()
.into_iter()
.flat_map(|it| {
it.iter().map(|x| {
x.to_path()
.expect("internal error: failed to convert lib path")
})
})
})
.map(|it| {
it.into_os_string()
.into_string()
.expect("internal error: failed to convert OS path")
})
.collect()
} else {
vec![]
};
.map(|it| {
it.into_os_string()
.into_string()
.expect("internal error: failed to convert OS path")
})
.collect()
} else {
vec![]
};
// Check if the base_dir is a folder
// rsplit_once removes the separator
@ -75,8 +84,7 @@ impl Completer for DotNuCompletion {
partial = base_dir_partial;
} else {
// Fetch the current folder
#[allow(deprecated)]
let current_folder = working_set.permanent_state.current_work_dir();
let current_folder = self.engine_state.current_work_dir();
is_current_folder = true;
// Add the current folder and the lib dirs into the
@ -95,8 +103,8 @@ impl Completer for DotNuCompletion {
&partial,
&search_dir,
options,
working_set.permanent_state,
stack,
self.engine_state.as_ref(),
&self.stack,
);
completions
.into_iter()
@ -116,13 +124,14 @@ impl Completer for DotNuCompletion {
.map(move |x| SemanticSuggestion {
suggestion: Suggestion {
value: x.1,
description: None,
style: x.2,
extra: None,
span: reedline::Span {
start: x.0.start - offset,
end: x.0.end - offset,
},
append_whitespace: true,
..Suggestion::default()
},
// TODO????
kind: None,
@ -130,6 +139,10 @@ impl Completer for DotNuCompletion {
})
.collect();
sort_suggestions(&prefix_str, output, options)
output
}
fn get_sort_by(&self) -> SortBy {
SortBy::LevenshteinDistance
}
}

71
crates/nu-cli/src/completions/file_completions.rs
View File

@ -1,24 +1,33 @@
use crate::completions::{
completion_common::{adjust_if_intermediate, complete_item, AdjustView},
Completer, CompletionOptions,
Completer, CompletionOptions, SortBy,
};
use nu_ansi_term::Style;
use nu_protocol::{
engine::{EngineState, Stack, StateWorkingSet},
Span,
levenshtein_distance, Span,
};
use nu_utils::IgnoreCaseExt;
use reedline::Suggestion;
use std::path::Path;
use std::{
path::{Path, MAIN_SEPARATOR as SEP},
sync::Arc,
};
use super::SemanticSuggestion;
#[derive(Clone, Default)]
pub struct FileCompletion {}
#[derive(Clone)]
pub struct FileCompletion {
engine_state: Arc<EngineState>,
stack: Stack,
}
impl FileCompletion {
pub fn new() -> Self {
Self::default()
pub fn new(engine_state: Arc<EngineState>, stack: Stack) -> Self {
Self {
engine_state,
stack,
}
}
}
@ -26,11 +35,10 @@ impl Completer for FileCompletion {
fn fetch(
&mut self,
working_set: &StateWorkingSet,
stack: &Stack,
prefix: Vec<u8>,
span: Span,
offset: usize,
_pos: usize,
_: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
let AdjustView {
@ -39,39 +47,68 @@ impl Completer for FileCompletion {
readjusted,
} = adjust_if_intermediate(&prefix, working_set, span);
#[allow(deprecated)]
let items: Vec<_> = complete_item(
let output: Vec<_> = complete_item(
readjusted,
span,
&prefix,
&working_set.permanent_state.current_work_dir(),
&self.engine_state.current_work_dir(),
options,
working_set.permanent_state,
stack,
self.engine_state.as_ref(),
&self.stack,
)
.into_iter()
.map(move |x| SemanticSuggestion {
suggestion: Suggestion {
value: x.1,
description: None,
style: x.2,
extra: None,
span: reedline::Span {
start: x.0.start - offset,
end: x.0.end - offset,
},
..Suggestion::default()
append_whitespace: false,
},
// TODO????
kind: None,
})
.collect();
// Sort results prioritizing the non hidden folders
output
}
// Sort results prioritizing the non hidden folders
fn sort(&self, items: Vec<SemanticSuggestion>, prefix: Vec<u8>) -> Vec<SemanticSuggestion> {
let prefix_str = String::from_utf8_lossy(&prefix).to_string();
// Sort items
let mut sorted_items = items;
match self.get_sort_by() {
SortBy::Ascending => {
sorted_items.sort_by(|a, b| {
// Ignore trailing slashes in folder names when sorting
a.suggestion
.value
.trim_end_matches(SEP)
.cmp(b.suggestion.value.trim_end_matches(SEP))
});
}
SortBy::LevenshteinDistance => {
sorted_items.sort_by(|a, b| {
let a_distance = levenshtein_distance(&prefix_str, &a.suggestion.value);
let b_distance = levenshtein_distance(&prefix_str, &b.suggestion.value);
a_distance.cmp(&b_distance)
});
}
_ => (),
}
// Separate the results between hidden and non hidden
let mut hidden: Vec<SemanticSuggestion> = vec![];
let mut non_hidden: Vec<SemanticSuggestion> = vec![];
for item in items.into_iter() {
for item in sorted_items.into_iter() {
let item_path = Path::new(&item.suggestion.value);
if let Some(value) = item_path.file_name() {

15
crates/nu-cli/src/completions/flag_completions.rs
View File

@ -1,7 +1,7 @@
use crate::completions::{completion_common::sort_suggestions, Completer, CompletionOptions};
use crate::completions::{Completer, CompletionOptions};
use nu_protocol::{
ast::{Expr, Expression},
engine::{Stack, StateWorkingSet},
engine::StateWorkingSet,
Span,
};
use reedline::Suggestion;
@ -23,11 +23,10 @@ impl Completer for FlagCompletion {
fn fetch(
&mut self,
working_set: &StateWorkingSet,
_stack: &Stack,
prefix: Vec<u8>,
span: Span,
offset: usize,
_pos: usize,
_: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
// Check if it's a flag
@ -49,12 +48,13 @@ impl Completer for FlagCompletion {
suggestion: Suggestion {
value: String::from_utf8_lossy(&named).to_string(),
description: Some(flag_desc.to_string()),
style: None,
extra: None,
span: reedline::Span {
start: span.start - offset,
end: span.end - offset,
},
append_whitespace: true,
..Suggestion::default()
},
// TODO????
kind: None,
@ -75,12 +75,13 @@ impl Completer for FlagCompletion {
suggestion: Suggestion {
value: String::from_utf8_lossy(&named).to_string(),
description: Some(flag_desc.to_string()),
style: None,
extra: None,
span: reedline::Span {
start: span.start - offset,
end: span.end - offset,
},
append_whitespace: true,
..Suggestion::default()
},
// TODO????
kind: None,
@ -88,7 +89,7 @@ impl Completer for FlagCompletion {
}
}
return sort_suggestions(&String::from_utf8_lossy(&prefix), output, options);
return output;
}
vec![]

2
crates/nu-cli/src/completions/mod.rs
View File

@ -13,7 +13,7 @@ mod variable_completions;
pub use base::{Completer, SemanticSuggestion, SuggestionKind};
pub use command_completions::CommandCompletion;
pub use completer::NuCompleter;
pub use completion_options::{CompletionOptions, MatchAlgorithm};
pub use completion_options::{CompletionOptions, MatchAlgorithm, SortBy};
pub use custom_completions::CustomCompletion;
pub use directory_completions::DirectoryCompletion;
pub use dotnu_completions::DotNuCompletion;

107
crates/nu-cli/src/completions/variable_completions.rs
View File

@ -3,22 +3,30 @@ use crate::completions::{
};
use nu_engine::{column::get_columns, eval_variable};
use nu_protocol::{
engine::{Stack, StateWorkingSet},
engine::{EngineState, Stack, StateWorkingSet},
Span, Value,
};
use reedline::Suggestion;
use std::str;
use super::completion_common::sort_suggestions;
use std::{str, sync::Arc};
#[derive(Clone)]
pub struct VariableCompletion {
engine_state: Arc<EngineState>, // TODO: Is engine state necessary? It's already a part of working set in fetch()
stack: Stack,
var_context: (Vec<u8>, Vec<Vec<u8>>), // tuple with $var and the sublevels (.b.c.d)
}
impl VariableCompletion {
pub fn new(var_context: (Vec<u8>, Vec<Vec<u8>>)) -> Self {
Self { var_context }
pub fn new(
engine_state: Arc<EngineState>,
stack: Stack,
var_context: (Vec<u8>, Vec<Vec<u8>>),
) -> Self {
Self {
engine_state,
stack,
var_context,
}
}
}
@ -26,11 +34,10 @@ impl Completer for VariableCompletion {
fn fetch(
&mut self,
working_set: &StateWorkingSet,
stack: &Stack,
prefix: Vec<u8>,
span: Span,
offset: usize,
_pos: usize,
_: usize,
options: &CompletionOptions,
) -> Vec<SemanticSuggestion> {
let mut output = vec![];
@ -42,13 +49,12 @@ impl Completer for VariableCompletion {
end: span.end - offset,
};
let sublevels_count = self.var_context.1.len();
let prefix_str = String::from_utf8_lossy(&prefix);
// Completions for the given variable
if !var_str.is_empty() {
// Completion for $env.<tab>
if var_str == "$env" {
let env_vars = stack.get_env_vars(working_set.permanent_state);
let env_vars = self.stack.get_env_vars(&self.engine_state);
// Return nested values
if sublevels_count > 0 {
@ -72,7 +78,7 @@ impl Completer for VariableCompletion {
}
}
return sort_suggestions(&prefix_str, output, options);
return output;
}
} else {
// No nesting provided, return all env vars
@ -85,15 +91,18 @@ impl Completer for VariableCompletion {
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: env_var.0,
description: None,
style: None,
extra: None,
span: current_span,
..Suggestion::default()
append_whitespace: false,
},
kind: Some(SuggestionKind::Type(env_var.1.get_type())),
});
}
}
return sort_suggestions(&prefix_str, output, options);
return output;
}
}
@ -101,8 +110,8 @@ impl Completer for VariableCompletion {
if var_str == "$nu" {
// Eval nu var
if let Ok(nuval) = eval_variable(
working_set.permanent_state,
stack,
&self.engine_state,
&self.stack,
nu_protocol::NU_VARIABLE_ID,
nu_protocol::Span::new(current_span.start, current_span.end),
) {
@ -117,14 +126,14 @@ impl Completer for VariableCompletion {
}
}
return sort_suggestions(&prefix_str, output, options);
return output;
}
}
// Completion other variable types
if let Some(var_id) = var_id {
// Extract the variable value from the stack
let var = stack.get_var(var_id, Span::new(span.start, span.end));
let var = self.stack.get_var(var_id, Span::new(span.start, span.end));
// If the value exists and it's of type Record
if let Ok(value) = var {
@ -139,7 +148,7 @@ impl Completer for VariableCompletion {
}
}
return sort_suggestions(&prefix_str, output, options);
return output;
}
}
}
@ -154,8 +163,11 @@ impl Completer for VariableCompletion {
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: builtin.to_string(),
description: None,
style: None,
extra: None,
span: current_span,
..Suggestion::default()
append_whitespace: false,
},
// TODO is there a way to get the VarId to get the type???
kind: None,
@ -178,8 +190,11 @@ impl Completer for VariableCompletion {
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: String::from_utf8_lossy(v.0).to_string(),
description: None,
style: None,
extra: None,
span: current_span,
..Suggestion::default()
append_whitespace: false,
},
kind: Some(SuggestionKind::Type(
working_set.get_variable(*v.1).ty.clone(),
@ -192,11 +207,7 @@ impl Completer for VariableCompletion {
// Permanent state vars
// for scope in &self.engine_state.scope {
for overlay_frame in working_set
.permanent_state
.active_overlays(&removed_overlays)
.rev()
{
for overlay_frame in self.engine_state.active_overlays(&removed_overlays).rev() {
for v in &overlay_frame.vars {
if options.match_algorithm.matches_u8_insensitive(
options.case_sensitive,
@ -206,8 +217,11 @@ impl Completer for VariableCompletion {
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: String::from_utf8_lossy(v.0).to_string(),
description: None,
style: None,
extra: None,
span: current_span,
..Suggestion::default()
append_whitespace: false,
},
kind: Some(SuggestionKind::Type(
working_set.get_variable(*v.1).ty.clone(),
@ -217,8 +231,6 @@ impl Completer for VariableCompletion {
}
}
output = sort_suggestions(&prefix_str, output, options);
output.dedup(); // TODO: Removes only consecutive duplicates, is it intended?
output
@ -243,8 +255,29 @@ fn nested_suggestions(
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: col.clone(),
description: None,
style: None,
extra: None,
span: current_span,
..Suggestion::default()
append_whitespace: false,
},
kind: Some(kind.clone()),
});
}
output
}
Value::LazyRecord { val, .. } => {
// Add all the columns as completion
for column_name in val.column_names() {
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: column_name.to_string(),
description: None,
style: None,
extra: None,
span: current_span,
append_whitespace: false,
},
kind: Some(kind.clone()),
});
@ -257,8 +290,11 @@ fn nested_suggestions(
output.push(SemanticSuggestion {
suggestion: Suggestion {
value: column_name,
description: None,
style: None,
extra: None,
span: current_span,
..Suggestion::default()
append_whitespace: false,
},
kind: Some(kind.clone()),
});
@ -285,6 +321,17 @@ fn recursive_value(val: &Value, sublevels: &[Vec<u8>]) -> Result<Value, Span> {
Err(span)
}
}
Value::LazyRecord { val, .. } => {
for col in val.column_names() {
if col.as_bytes() == *sublevel {
let val = val.get_column_value(col).map_err(|_| span)?;
return recursive_value(&val, next_sublevels);
}
}
// Current sublevel value not found
Err(span)
}
Value::List { vals, .. } => {
for col in get_columns(vals.as_slice()) {
if col.as_bytes() == *sublevel {

127
crates/nu-cli/src/config_files.rs
View File

@ -1,14 +1,14 @@
use crate::util::eval_source;
#[cfg(feature = "plugin")]
use nu_path::canonicalize_with;
#[cfg(feature = "plugin")]
use nu_protocol::{engine::StateWorkingSet, report_error, ParseError, PluginRegistryFile, Spanned};
use nu_protocol::{
engine::{EngineState, Stack},
report_error_new, HistoryFileFormat, PipelineData,
engine::{EngineState, Stack, StateWorkingSet},
report_error, HistoryFileFormat, PipelineData,
};
#[cfg(feature = "plugin")]
use nu_utils::perf;
use nu_protocol::{ParseError, PluginRegistryFile, Spanned};
#[cfg(feature = "plugin")]
use nu_utils::utils::perf;
use std::path::PathBuf;
#[cfg(feature = "plugin")]
@ -25,9 +25,10 @@ pub fn read_plugin_file(
plugin_file: Option<Spanned<String>>,
storage_path: &str,
) {
use nu_protocol::ShellError;
use std::path::Path;
use nu_protocol::{report_error_new, ShellError};
let span = plugin_file.as_ref().map(|s| s.span);
// Check and warn + abort if this is a .nu plugin file
@ -53,10 +54,13 @@ pub fn read_plugin_file(
// Reading signatures from plugin registry file
// The plugin.msgpackz file stores the parsed signature collected from each registered plugin
add_plugin_file(engine_state, plugin_file.clone(), storage_path);
perf!(
perf(
"add plugin file to engine_state",
start_time,
engine_state.get_config().use_ansi_coloring
file!(),
line!(),
column!(),
engine_state.get_config().use_ansi_coloring,
);
start_time = std::time::Instant::now();
@ -134,26 +138,32 @@ pub fn read_plugin_file(
}
};
perf!(
perf(
&format!("read plugin file {}", plugin_path.display()),
start_time,
engine_state.get_config().use_ansi_coloring
file!(),
line!(),
column!(),
engine_state.get_config().use_ansi_coloring,
);
start_time = std::time::Instant::now();
let mut working_set = StateWorkingSet::new(engine_state);
nu_plugin_engine::load_plugin_file(&mut working_set, &contents, span);
nu_plugin::load_plugin_file(&mut working_set, &contents, span);
if let Err(err) = engine_state.merge_delta(working_set.render()) {
report_error_new(engine_state, &err);
return;
}
perf!(
perf(
&format!("load plugin file {}", plugin_path.display()),
start_time,
engine_state.get_config().use_ansi_coloring
file!(),
line!(),
column!(),
engine_state.get_config().use_ansi_coloring,
);
}
}
@ -167,37 +177,35 @@ pub fn add_plugin_file(
use std::path::Path;
let working_set = StateWorkingSet::new(engine_state);
let cwd = working_set.get_cwd();
if let Ok(cwd) = engine_state.cwd_as_string(None) {
if let Some(plugin_file) = plugin_file {
let path = Path::new(&plugin_file.item);
let path_dir = path.parent().unwrap_or(path);
// Just try to canonicalize the directory of the plugin file first.
if let Ok(path_dir) = canonicalize_with(path_dir, &cwd) {
// Try to canonicalize the actual filename, but it's ok if that fails. The file doesn't
// have to exist.
let path = path_dir.join(path.file_name().unwrap_or(path.as_os_str()));
let path = canonicalize_with(&path, &cwd).unwrap_or(path);
engine_state.plugin_path = Some(path)
} else {
// It's an error if the directory for the plugin file doesn't exist.
report_error(
&working_set,
&ParseError::FileNotFound(
path_dir.to_string_lossy().into_owned(),
plugin_file.span,
),
);
}
} else if let Some(mut plugin_path) = nu_path::config_dir() {
// Path to store plugins signatures
plugin_path.push(storage_path);
let mut plugin_path =
canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path.into());
plugin_path.push(PLUGIN_FILE);
let plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
engine_state.plugin_path = Some(plugin_path);
if let Some(plugin_file) = plugin_file {
let path = Path::new(&plugin_file.item);
let path_dir = path.parent().unwrap_or(path);
// Just try to canonicalize the directory of the plugin file first.
if let Ok(path_dir) = canonicalize_with(path_dir, &cwd) {
// Try to canonicalize the actual filename, but it's ok if that fails. The file doesn't
// have to exist.
let path = path_dir.join(path.file_name().unwrap_or(path.as_os_str()));
let path = canonicalize_with(&path, &cwd).unwrap_or(path);
engine_state.plugin_path = Some(path)
} else {
// It's an error if the directory for the plugin file doesn't exist.
report_error(
&working_set,
&ParseError::FileNotFound(
path_dir.to_string_lossy().into_owned(),
plugin_file.span,
),
);
}
} else if let Some(mut plugin_path) = nu_path::config_dir() {
// Path to store plugins signatures
plugin_path.push(storage_path);
let mut plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
plugin_path.push(PLUGIN_FILE);
let plugin_path = canonicalize_with(&plugin_path, &cwd).unwrap_or(plugin_path);
engine_state.plugin_path = Some(plugin_path);
}
}
@ -227,14 +235,16 @@ pub fn eval_config_contents(
engine_state.file = prev_file;
// Merge the environment in case env vars changed in the config
match engine_state.cwd(Some(stack)) {
match nu_engine::env::current_dir(engine_state, stack) {
Ok(cwd) => {
if let Err(e) = engine_state.merge_env(stack, cwd) {
report_error_new(engine_state, &e);
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &e);
}
}
Err(e) => {
report_error_new(engine_state, &e);
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &e);
}
}
}
@ -248,23 +258,21 @@ pub(crate) fn get_history_path(storage_path: &str, mode: HistoryFileFormat) -> O
HistoryFileFormat::PlainText => HISTORY_FILE_TXT,
HistoryFileFormat::Sqlite => HISTORY_FILE_SQLITE,
});
history_path.into()
history_path
})
}
#[cfg(feature = "plugin")]
pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -> bool {
use nu_protocol::{
PluginExample, PluginIdentity, PluginRegistryItem, PluginRegistryItemData, PluginSignature,
ShellError,
report_error_new, PluginExample, PluginIdentity, PluginRegistryItem,
PluginRegistryItemData, PluginSignature, ShellError,
};
use std::collections::BTreeMap;
let start_time = std::time::Instant::now();
let Ok(cwd) = engine_state.cwd_as_string(None) else {
return false;
};
let cwd = engine_state.current_work_dir();
let Some(config_dir) = nu_path::config_dir().and_then(|mut dir| {
dir.push(storage_path);
@ -298,15 +306,14 @@ pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -
let mut engine_state = engine_state.clone();
let mut stack = Stack::new();
if eval_source(
if !eval_source(
&mut engine_state,
&mut stack,
&old_contents,
&old_plugin_file_path.to_string_lossy(),
PipelineData::Empty,
false,
) != 0
{
) {
return false;
}
@ -336,10 +343,7 @@ pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -
name: identity.name().to_owned(),
filename: identity.filename().to_owned(),
shell: identity.shell().map(|p| p.to_owned()),
data: PluginRegistryItemData::Valid {
metadata: Default::default(),
commands,
},
data: PluginRegistryItemData::Valid { commands },
});
}
@ -373,10 +377,13 @@ pub fn migrate_old_plugin_file(engine_state: &EngineState, storage_path: &str) -
);
}
perf!(
perf(
"migrate old plugin file",
start_time,
engine_state.get_config().use_ansi_coloring
file!(),
line!(),
column!(),
engine_state.get_config().use_ansi_coloring,
);
true
}

99
crates/nu-cli/src/eval_cmds.rs
View File

@ -1,19 +1,12 @@
use log::info;
use miette::Result;
use nu_engine::{convert_env_values, eval_block};
use nu_parser::parse;
use nu_protocol::{
debugger::WithoutDebug,
engine::{EngineState, Stack, StateWorkingSet},
report_error, PipelineData, ShellError, Spanned, Value,
report_error, PipelineData, Spanned, Value,
};
use std::sync::Arc;
#[derive(Default)]
pub struct EvaluateCommandsOpts {
pub table_mode: Option<Value>,
pub error_style: Option<Value>,
pub no_newline: bool,
}
/// Run a command (or commands) given to us by the user
pub fn evaluate_commands(
@ -21,40 +14,22 @@ pub fn evaluate_commands(
engine_state: &mut EngineState,
stack: &mut Stack,
input: PipelineData,
opts: EvaluateCommandsOpts,
) -> Result<(), ShellError> {
let EvaluateCommandsOpts {
table_mode,
error_style,
no_newline,
} = opts;
// Handle the configured error style early
if let Some(e_style) = error_style {
match e_style.coerce_str()?.parse() {
Ok(e_style) => {
Arc::make_mut(&mut engine_state.config).error_style = e_style;
}
Err(err) => {
return Err(ShellError::GenericError {
error: "Invalid value for `--error-style`".into(),
msg: err.into(),
span: Some(e_style.span()),
help: None,
inner: vec![],
});
}
}
}
table_mode: Option<Value>,
no_newline: bool,
) -> Result<Option<i64>> {
// Translate environment variables from Strings to Values
convert_env_values(engine_state, stack)?;
if let Some(e) = convert_env_values(engine_state, stack) {
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &e);
std::process::exit(1);
}
// Parse the source code
let (block, delta) = {
if let Some(ref t_mode) = table_mode {
Arc::make_mut(&mut engine_state.config).table_mode =
t_mode.coerce_str()?.parse().unwrap_or_default();
let mut config = engine_state.get_config().clone();
config.table_mode = t_mode.coerce_str()?.parse().unwrap_or_default();
engine_state.set_config(config);
}
let mut working_set = StateWorkingSet::new(engine_state);
@ -66,39 +41,43 @@ pub fn evaluate_commands(
if let Some(err) = working_set.parse_errors.first() {
report_error(&working_set, err);
std::process::exit(1);
}
if let Some(err) = working_set.compile_errors.first() {
report_error(&working_set, err);
// Not a fatal error, for now
std::process::exit(1);
}
(output, working_set.render())
};
// Update permanent state
engine_state.merge_delta(delta)?;
if let Err(err) = engine_state.merge_delta(delta) {
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &err);
}
// Run the block
let pipeline = eval_block::<WithoutDebug>(engine_state, stack, &block, input)?;
if let PipelineData::Value(Value::Error { error, .. }, ..) = pipeline {
return Err(*error);
}
if let Some(t_mode) = table_mode {
Arc::make_mut(&mut engine_state.config).table_mode =
t_mode.coerce_str()?.parse().unwrap_or_default();
}
if let Some(status) = pipeline.print(engine_state, stack, no_newline, false)? {
if status.code() != 0 {
std::process::exit(status.code())
let exit_code = match eval_block::<WithoutDebug>(engine_state, stack, &block, input) {
Ok(pipeline_data) => {
let mut config = engine_state.get_config().clone();
if let Some(t_mode) = table_mode {
config.table_mode = t_mode.coerce_str()?.parse().unwrap_or_default();
}
crate::eval_file::print_table_or_error(
engine_state,
stack,
pipeline_data,
&mut config,
no_newline,
)
}
}
Err(err) => {
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &err);
std::process::exit(1);
}
};
info!("evaluate {}:{}:{}", file!(), line!(), column!());
Ok(())
Ok(exit_code)
}

220
crates/nu-cli/src/eval_file.rs
View File

@ -1,14 +1,15 @@
use crate::util::eval_source;
use log::{info, trace};
use nu_engine::{convert_env_values, eval_block};
use miette::{IntoDiagnostic, Result};
use nu_engine::{convert_env_values, current_dir, eval_block};
use nu_parser::parse;
use nu_path::canonicalize_with;
use nu_protocol::{
debugger::WithoutDebug,
engine::{EngineState, Stack, StateWorkingSet},
report_error, PipelineData, ShellError, Span, Value,
report_error, Config, PipelineData, ShellError, Span, Value,
};
use std::sync::Arc;
use std::{io::Write, sync::Arc};
/// Entry point for evaluating a file.
///
@ -20,40 +21,73 @@ pub fn evaluate_file(
engine_state: &mut EngineState,
stack: &mut Stack,
input: PipelineData,
) -> Result<(), ShellError> {
) -> Result<()> {
// Convert environment variables from Strings to Values and store them in the engine state.
convert_env_values(engine_state, stack)?;
if let Some(e) = convert_env_values(engine_state, stack) {
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &e);
std::process::exit(1);
}
let cwd = engine_state.cwd_as_string(Some(stack))?;
let cwd = current_dir(engine_state, stack)?;
let file_path =
canonicalize_with(&path, cwd).map_err(|err| ShellError::FileNotFoundCustom {
msg: format!("Could not access file '{path}': {err}"),
span: Span::unknown(),
})?;
let file_path = canonicalize_with(&path, cwd).unwrap_or_else(|e| {
let working_set = StateWorkingSet::new(engine_state);
report_error(
&working_set,
&ShellError::FileNotFoundCustom {
msg: format!("Could not access file '{}': {:?}", path, e.to_string()),
span: Span::unknown(),
},
);
std::process::exit(1);
});
let file_path_str = file_path
.to_str()
.ok_or_else(|| ShellError::NonUtf8Custom {
msg: format!(
"Input file name '{}' is not valid UTF8",
file_path.to_string_lossy()
),
span: Span::unknown(),
})?;
let file_path_str = file_path.to_str().unwrap_or_else(|| {
let working_set = StateWorkingSet::new(engine_state);
report_error(
&working_set,
&ShellError::NonUtf8Custom {
msg: format!(
"Input file name '{}' is not valid UTF8",
file_path.to_string_lossy()
),
span: Span::unknown(),
},
);
std::process::exit(1);
});
let file = std::fs::read(&file_path).map_err(|err| ShellError::FileNotFoundCustom {
msg: format!("Could not read file '{file_path_str}': {err}"),
span: Span::unknown(),
})?;
let file = std::fs::read(&file_path)
.into_diagnostic()
.unwrap_or_else(|e| {
let working_set = StateWorkingSet::new(engine_state);
report_error(
&working_set,
&ShellError::FileNotFoundCustom {
msg: format!(
"Could not read file '{}': {:?}",
file_path_str,
e.to_string()
),
span: Span::unknown(),
},
);
std::process::exit(1);
});
engine_state.file = Some(file_path.clone());
let parent = file_path
.parent()
.ok_or_else(|| ShellError::FileNotFoundCustom {
msg: format!("The file path '{file_path_str}' does not have a parent"),
span: Span::unknown(),
})?;
let parent = file_path.parent().unwrap_or_else(|| {
let working_set = StateWorkingSet::new(engine_state);
report_error(
&working_set,
&ShellError::FileNotFoundCustom {
msg: format!("The file path '{file_path_str}' does not have a parent"),
span: Span::unknown(),
},
);
std::process::exit(1);
});
stack.add_env_var(
"FILE_PWD".to_string(),
@ -76,21 +110,12 @@ pub fn evaluate_file(
trace!("parsing file: {}", file_path_str);
let block = parse(&mut working_set, Some(file_path_str), &file, false);
if let Some(warning) = working_set.parse_warnings.first() {
report_error(&working_set, warning);
}
// If any parse errors were found, report the first error and exit.
if let Some(err) = working_set.parse_errors.first() {
report_error(&working_set, err);
std::process::exit(1);
}
if let Some(err) = working_set.compile_errors.first() {
report_error(&working_set, err);
// Not a fatal error, for now
}
// Look for blocks whose name starts with "main" and replace it with the filename.
for block in working_set.delta.blocks.iter_mut().map(Arc::make_mut) {
if block.signature.name == "main" {
@ -102,48 +127,119 @@ pub fn evaluate_file(
}
// Merge the changes into the engine state.
engine_state.merge_delta(working_set.delta)?;
engine_state
.merge_delta(working_set.delta)
.expect("merging delta into engine_state should succeed");
// Check if the file contains a main command.
let exit_code = if engine_state.find_decl(b"main", &[]).is_some() {
if engine_state.find_decl(b"main", &[]).is_some() {
// Evaluate the file, but don't run main yet.
let pipeline =
match eval_block::<WithoutDebug>(engine_state, stack, &block, PipelineData::empty()) {
Ok(data) => data,
Err(ShellError::Return { .. }) => {
// Allow early return before main is run.
return Ok(());
}
Err(err) => return Err(err),
};
let pipeline_data =
eval_block::<WithoutDebug>(engine_state, stack, &block, PipelineData::empty());
let pipeline_data = match pipeline_data {
Err(ShellError::Return { .. }) => {
// Allow early return before main is run.
return Ok(());
}
x => x,
}
.unwrap_or_else(|e| {
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &e);
std::process::exit(1);
});
// Print the pipeline output of the last command of the file.
if let Some(status) = pipeline.print(engine_state, stack, true, false)? {
if status.code() != 0 {
std::process::exit(status.code())
// Print the pipeline output of the file.
// The pipeline output of a file is the pipeline output of its last command.
let result = pipeline_data.print(engine_state, stack, true, false);
match result {
Err(err) => {
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &err);
std::process::exit(1);
}
Ok(exit_code) => {
if exit_code != 0 {
std::process::exit(exit_code as i32);
}
}
}
// Invoke the main command with arguments.
// Arguments with whitespaces are quoted, thus can be safely concatenated by whitespace.
let args = format!("main {}", args.join(" "));
eval_source(
if !eval_source(
engine_state,
stack,
args.as_bytes(),
"<commandline>",
input,
true,
)
} else {
eval_source(engine_state, stack, &file, file_path_str, input, true)
};
if exit_code != 0 {
std::process::exit(exit_code)
) {
std::process::exit(1);
}
} else if !eval_source(engine_state, stack, &file, file_path_str, input, true) {
std::process::exit(1);
}
info!("evaluate {}:{}:{}", file!(), line!(), column!());
Ok(())
}
pub(crate) fn print_table_or_error(
engine_state: &mut EngineState,
stack: &mut Stack,
mut pipeline_data: PipelineData,
config: &mut Config,
no_newline: bool,
) -> Option<i64> {
let exit_code = match &mut pipeline_data {
PipelineData::ExternalStream { exit_code, .. } => exit_code.take(),
_ => None,
};
// Change the engine_state config to use the passed in configuration
engine_state.set_config(config.clone());
if let PipelineData::Value(Value::Error { error, .. }, ..) = &pipeline_data {
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &**error);
std::process::exit(1);
}
// We don't need to do anything special to print a table because print() handles it
print_or_exit(pipeline_data, engine_state, stack, no_newline);
// Make sure everything has finished
if let Some(exit_code) = exit_code {
let mut exit_code: Vec<_> = exit_code.into_iter().collect();
exit_code
.pop()
.and_then(|last_exit_code| match last_exit_code {
Value::Int { val: code, .. } => Some(code),
_ => None,
})
} else {
None
}
}
fn print_or_exit(
pipeline_data: PipelineData,
engine_state: &EngineState,
stack: &mut Stack,
no_newline: bool,
) {
let result = pipeline_data.print(engine_state, stack, no_newline, false);
let _ = std::io::stdout().flush();
let _ = std::io::stderr().flush();
if let Err(error) = result {
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &error);
let _ = std::io::stderr().flush();
std::process::exit(1);
}
}

3
crates/nu-cli/src/lib.rs
View File

@ -1,4 +1,3 @@
#![doc = include_str!("../README.md")]
mod commands;
mod completions;
mod config_files;
@ -18,7 +17,7 @@ mod validation;
pub use commands::add_cli_context;
pub use completions::{FileCompletion, NuCompleter, SemanticSuggestion, SuggestionKind};
pub use config_files::eval_config_contents;
pub use eval_cmds::{evaluate_commands, EvaluateCommandsOpts};
pub use eval_cmds::evaluate_commands;
pub use eval_file::evaluate_file;
pub use menus::NuHelpCompleter;
pub use nu_cmd_base::util::get_init_cwd;

73
crates/nu-cli/src/menus/help_completions.rs
View File

@ -1,70 +1,62 @@
use nu_engine::documentation::{get_flags_section, HelpStyle};
use nu_protocol::{engine::EngineState, levenshtein_distance, Config};
use nu_engine::documentation::get_flags_section;
use nu_protocol::{engine::EngineState, levenshtein_distance};
use nu_utils::IgnoreCaseExt;
use reedline::{Completer, Suggestion};
use std::{fmt::Write, sync::Arc};
pub struct NuHelpCompleter {
engine_state: Arc<EngineState>,
config: Arc<Config>,
}
pub struct NuHelpCompleter(Arc<EngineState>);
impl NuHelpCompleter {
pub fn new(engine_state: Arc<EngineState>, config: Arc<Config>) -> Self {
Self {
engine_state,
config,
}
pub fn new(engine_state: Arc<EngineState>) -> Self {
Self(engine_state)
}
fn completion_helper(&self, line: &str, pos: usize) -> Vec<Suggestion> {
let full_commands = self.0.get_signatures_with_examples(false);
let folded_line = line.to_folded_case();
let mut help_style = HelpStyle::default();
help_style.update_from_config(&self.engine_state, &self.config);
let mut commands = self
.engine_state
.get_decls_sorted(false)
.into_iter()
.filter_map(|(_, decl_id)| {
let decl = self.engine_state.get_decl(decl_id);
(decl.name().to_folded_case().contains(&folded_line)
|| decl.usage().to_folded_case().contains(&folded_line)
|| decl
.search_terms()
.into_iter()
//Vec<(Signature, Vec<Example>, bool, bool)> {
let mut commands = full_commands
.iter()
.filter(|(sig, _, _, _, _)| {
sig.name.to_folded_case().contains(&folded_line)
|| sig.usage.to_folded_case().contains(&folded_line)
|| sig
.search_terms
.iter()
.any(|term| term.to_folded_case().contains(&folded_line))
|| decl.extra_usage().to_folded_case().contains(&folded_line))
.then_some(decl)
|| sig.extra_usage.to_folded_case().contains(&folded_line)
})
.collect::<Vec<_>>();
commands.sort_by_cached_key(|decl| levenshtein_distance(line, decl.name()));
commands.sort_by(|(a, _, _, _, _), (b, _, _, _, _)| {
let a_distance = levenshtein_distance(line, &a.name);
let b_distance = levenshtein_distance(line, &b.name);
a_distance.cmp(&b_distance)
});
commands
.into_iter()
.map(|decl| {
.map(|(sig, examples, _, _, _)| {
let mut long_desc = String::new();
let usage = decl.usage();
let usage = &sig.usage;
if !usage.is_empty() {
long_desc.push_str(usage);
long_desc.push_str("\r\n\r\n");
}
let extra_usage = decl.extra_usage();
let extra_usage = &sig.extra_usage;
if !extra_usage.is_empty() {
long_desc.push_str(extra_usage);
long_desc.push_str("\r\n\r\n");
}
let sig = decl.signature();
let _ = write!(long_desc, "Usage:\r\n > {}\r\n", sig.call_signature());
if !sig.named.is_empty() {
long_desc.push_str(&get_flags_section(&sig, &help_style, |v| {
v.to_parsable_string(", ", &self.config)
long_desc.push_str(&get_flags_section(Some(&*self.0.clone()), sig, |v| {
v.to_parsable_string(", ", &self.0.config)
}))
}
@ -80,7 +72,7 @@ impl NuHelpCompleter {
let opt_suffix = if let Some(value) = &positional.default_value {
format!(
" (optional, default: {})",
&value.to_parsable_string(", ", &self.config),
&value.to_parsable_string(", ", &self.0.config),
)
} else {
(" (optional)").to_string()
@ -101,21 +93,21 @@ impl NuHelpCompleter {
}
}
let extra: Vec<String> = decl
.examples()
let extra: Vec<String> = examples
.iter()
.map(|example| example.example.replace('\n', "\r\n"))
.collect();
Suggestion {
value: decl.name().into(),
value: sig.name.clone(),
description: Some(long_desc),
style: None,
extra: Some(extra),
span: reedline::Span {
start: pos - line.len(),
end: pos,
},
..Suggestion::default()
append_whitespace: false,
}
})
.collect()
@ -146,8 +138,7 @@ mod test {
) {
let engine_state =
nu_command::add_shell_command_context(nu_cmd_lang::create_default_context());
let config = engine_state.get_config().clone();
let mut completer = NuHelpCompleter::new(engine_state.into(), config);
let mut completer = NuHelpCompleter::new(engine_state.into());
let suggestions = completer.complete(line, end);
assert_eq!(

11
crates/nu-cli/src/menus/menu_completions.rs
View File

@ -59,7 +59,8 @@ impl Completer for NuMenuCompleter {
let res = eval_block::<WithoutDebug>(&self.engine_state, &mut self.stack, block, input);
if let Ok(values) = res.and_then(|data| data.into_value(self.span)) {
if let Ok(values) = res {
let values = values.into_value(self.span);
convert_to_suggestions(values, line, pos, self.only_buffer_difference)
} else {
Vec::new()
@ -142,9 +143,10 @@ fn convert_to_suggestions(
vec![Suggestion {
value: text,
description,
style: None,
extra,
span,
..Suggestion::default()
append_whitespace: false,
}]
}
Value::List { vals, .. } => vals
@ -153,6 +155,9 @@ fn convert_to_suggestions(
.collect(),
_ => vec![Suggestion {
value: format!("Not a record: {value:?}"),
description: None,
style: None,
extra: None,
span: reedline::Span {
start: if only_buffer_difference {
pos - line.len()
@ -165,7 +170,7 @@ fn convert_to_suggestions(
line.len()
},
},
..Suggestion::default()
append_whitespace: false,
}],
}
}

13
crates/nu-cli/src/nu_highlight.rs
View File

@ -1,5 +1,3 @@
use std::sync::Arc;
use nu_engine::command_prelude::*;
use reedline::{Highlighter, StyledText};
@ -34,11 +32,14 @@ impl Command for NuHighlight {
) -> Result<PipelineData, ShellError> {
let head = call.head;
let signals = engine_state.signals();
let ctrlc = engine_state.ctrlc.clone();
let engine_state = std::sync::Arc::new(engine_state.clone());
let config = engine_state.get_config().clone();
let highlighter = crate::NuHighlighter {
engine_state: Arc::new(engine_state.clone()),
stack: Arc::new(stack.clone()),
engine_state,
stack: std::sync::Arc::new(stack.clone()),
config,
};
input.map(
@ -49,7 +50,7 @@ impl Command for NuHighlight {
}
Err(err) => Value::error(err, head),
},
signals,
ctrlc,
)
}

39
crates/nu-cli/src/prompt.rs
View File

@ -1,10 +1,4 @@
use crate::prompt_update::{
POST_PROMPT_MARKER, PRE_PROMPT_MARKER, VSCODE_POST_PROMPT_MARKER, VSCODE_PRE_PROMPT_MARKER,
};
use nu_protocol::{
engine::{EngineState, Stack},
Value,
};
use crate::prompt_update::{POST_PROMPT_MARKER, PRE_PROMPT_MARKER};
#[cfg(windows)]
use nu_utils::enable_vt_processing;
use reedline::{
@ -16,8 +10,7 @@ use std::borrow::Cow;
/// Nushell prompt definition
#[derive(Clone)]
pub struct NushellPrompt {
shell_integration_osc133: bool,
shell_integration_osc633: bool,
shell_integration: bool,
left_prompt_string: Option<String>,
right_prompt_string: Option<String>,
default_prompt_indicator: Option<String>,
@ -25,20 +18,12 @@ pub struct NushellPrompt {
default_vi_normal_prompt_indicator: Option<String>,
default_multiline_indicator: Option<String>,
render_right_prompt_on_last_line: bool,
engine_state: EngineState,
stack: Stack,
}
impl NushellPrompt {
pub fn new(
shell_integration_osc133: bool,
shell_integration_osc633: bool,
engine_state: EngineState,
stack: Stack,
) -> NushellPrompt {
pub fn new(shell_integration: bool) -> NushellPrompt {
NushellPrompt {
shell_integration_osc133,
shell_integration_osc633,
shell_integration,
left_prompt_string: None,
right_prompt_string: None,
default_prompt_indicator: None,
@ -46,8 +31,6 @@ impl NushellPrompt {
default_vi_normal_prompt_indicator: None,
default_multiline_indicator: None,
render_right_prompt_on_last_line: false,
engine_state,
stack,
}
}
@ -123,19 +106,7 @@ impl Prompt for NushellPrompt {
.to_string()
.replace('\n', "\r\n");
if self.shell_integration_osc633 {
if self.stack.get_env_var(&self.engine_state, "TERM_PROGRAM")
== Some(Value::test_string("vscode"))
{
// We're in vscode and we have osc633 enabled
format!("{VSCODE_PRE_PROMPT_MARKER}{prompt}{VSCODE_POST_PROMPT_MARKER}").into()
} else if self.shell_integration_osc133 {
// If we're in VSCode but we don't find the env var, but we have osc133 set, then use it
format!("{PRE_PROMPT_MARKER}{prompt}{POST_PROMPT_MARKER}").into()
} else {
prompt.into()
}
} else if self.shell_integration_osc133 {
if self.shell_integration {
format!("{PRE_PROMPT_MARKER}{prompt}{POST_PROMPT_MARKER}").into()
} else {
prompt.into()

62
crates/nu-cli/src/prompt_update.rs
View File

@ -2,8 +2,8 @@ use crate::NushellPrompt;
use log::trace;
use nu_engine::ClosureEvalOnce;
use nu_protocol::{
engine::{EngineState, Stack},
report_error_new, Config, PipelineData, Value,
engine::{EngineState, Stack, StateWorkingSet},
report_error, Config, PipelineData, Value,
};
use reedline::Prompt;
@ -23,37 +23,10 @@ pub(crate) const TRANSIENT_PROMPT_INDICATOR_VI_NORMAL: &str =
"TRANSIENT_PROMPT_INDICATOR_VI_NORMAL";
pub(crate) const TRANSIENT_PROMPT_MULTILINE_INDICATOR: &str =
"TRANSIENT_PROMPT_MULTILINE_INDICATOR";
// Store all these Ansi Escape Markers here so they can be reused easily
// According to Daniel Imms @Tyriar, we need to do these this way:
// <133 A><prompt><133 B><command><133 C><command output>
pub(crate) const PRE_PROMPT_MARKER: &str = "\x1b]133;A\x1b\\";
pub(crate) const POST_PROMPT_MARKER: &str = "\x1b]133;B\x1b\\";
pub(crate) const PRE_EXECUTION_MARKER: &str = "\x1b]133;C\x1b\\";
#[allow(dead_code)]
pub(crate) const POST_EXECUTION_MARKER_PREFIX: &str = "\x1b]133;D;";
#[allow(dead_code)]
pub(crate) const POST_EXECUTION_MARKER_SUFFIX: &str = "\x1b\\";
// OSC633 is the same as OSC133 but specifically for VSCode
pub(crate) const VSCODE_PRE_PROMPT_MARKER: &str = "\x1b]633;A\x1b\\";
pub(crate) const VSCODE_POST_PROMPT_MARKER: &str = "\x1b]633;B\x1b\\";
#[allow(dead_code)]
pub(crate) const VSCODE_PRE_EXECUTION_MARKER: &str = "\x1b]633;C\x1b\\";
#[allow(dead_code)]
//"\x1b]633;D;{}\x1b\\"
pub(crate) const VSCODE_POST_EXECUTION_MARKER_PREFIX: &str = "\x1b]633;D;";
#[allow(dead_code)]
pub(crate) const VSCODE_POST_EXECUTION_MARKER_SUFFIX: &str = "\x1b\\";
#[allow(dead_code)]
pub(crate) const VSCODE_COMMANDLINE_MARKER: &str = "\x1b]633;E\x1b\\";
#[allow(dead_code)]
// "\x1b]633;P;Cwd={}\x1b\\"
pub(crate) const VSCODE_CWD_PROPERTY_MARKER_PREFIX: &str = "\x1b]633;P;Cwd=";
#[allow(dead_code)]
pub(crate) const VSCODE_CWD_PROPERTY_MARKER_SUFFIX: &str = "\x1b\\";
pub(crate) const RESET_APPLICATION_MODE: &str = "\x1b[?1l";
fn get_prompt_string(
prompt: &str,
@ -65,7 +38,7 @@ fn get_prompt_string(
.get_env_var(engine_state, prompt)
.and_then(|v| match v {
Value::Closure { val, .. } => {
let result = ClosureEvalOnce::new(engine_state, stack, *val)
let result = ClosureEvalOnce::new(engine_state, stack, val)
.run_with_input(PipelineData::Empty);
trace!(
@ -77,7 +50,8 @@ fn get_prompt_string(
result
.map_err(|err| {
report_error_new(engine_state, &err);
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &err);
})
.ok()
}
@ -107,34 +81,20 @@ pub(crate) fn update_prompt(
stack: &mut Stack,
nu_prompt: &mut NushellPrompt,
) {
let configured_left_prompt_string =
match get_prompt_string(PROMPT_COMMAND, config, engine_state, stack) {
Some(s) => s,
None => "".to_string(),
};
let left_prompt_string = get_prompt_string(PROMPT_COMMAND, config, engine_state, stack);
// Now that we have the prompt string lets ansify it.
// <133 A><prompt><133 B><command><133 C><command output>
let left_prompt_string = if config.shell_integration_osc633 {
if stack.get_env_var(engine_state, "TERM_PROGRAM") == Some(Value::test_string("vscode")) {
// We're in vscode and we have osc633 enabled
let left_prompt_string = if config.shell_integration {
if let Some(prompt_string) = left_prompt_string {
Some(format!(
"{VSCODE_PRE_PROMPT_MARKER}{configured_left_prompt_string}{VSCODE_POST_PROMPT_MARKER}"
))
} else if config.shell_integration_osc133 {
// If we're in VSCode but we don't find the env var, but we have osc133 set, then use it
Some(format!(
"{PRE_PROMPT_MARKER}{configured_left_prompt_string}{POST_PROMPT_MARKER}"
"{PRE_PROMPT_MARKER}{prompt_string}{POST_PROMPT_MARKER}"
))
} else {
configured_left_prompt_string.into()
left_prompt_string
}
} else if config.shell_integration_osc133 {
Some(format!(
"{PRE_PROMPT_MARKER}{configured_left_prompt_string}{POST_PROMPT_MARKER}"
))
} else {
configured_left_prompt_string.into()
left_prompt_string
};
let right_prompt_string = get_prompt_string(PROMPT_COMMAND_RIGHT, config, engine_state, stack);

279
crates/nu-cli/src/reedline_config.rs
View File

@ -1,6 +1,6 @@
use crate::{menus::NuMenuCompleter, NuHelpCompleter};
use crossterm::event::{KeyCode, KeyModifiers};
use nu_ansi_term::Style;
use log::trace;
use nu_color_config::{color_record_to_nustyle, lookup_ansi_color_style};
use nu_engine::eval_block;
use nu_parser::parse;
@ -75,21 +75,15 @@ const DEFAULT_HELP_MENU: &str = r#"
// Adds all menus to line editor
pub(crate) fn add_menus(
mut line_editor: Reedline,
engine_state_ref: Arc<EngineState>,
engine_state: Arc<EngineState>,
stack: &Stack,
config: Arc<Config>,
config: &Config,
) -> Result<Reedline, ShellError> {
//log::trace!("add_menus: config: {:#?}", &config);
trace!("add_menus: config: {:#?}", &config);
line_editor = line_editor.clear_menus();
for menu in &config.menus {
line_editor = add_menu(
line_editor,
menu,
engine_state_ref.clone(),
stack,
config.clone(),
)?
line_editor = add_menu(line_editor, menu, engine_state.clone(), stack, config)?
}
// Checking if the default menus have been added from the config file
@ -99,16 +93,13 @@ pub(crate) fn add_menus(
("help_menu", DEFAULT_HELP_MENU),
];
let mut engine_state = (*engine_state_ref).clone();
let mut menu_eval_results = vec![];
for (name, definition) in default_menus {
if !config
.menus
.iter()
.any(|menu| menu.name.to_expanded_string("", &config) == name)
.any(|menu| menu.name.to_expanded_string("", config) == name)
{
let (block, delta) = {
let (block, _) = {
let mut working_set = StateWorkingSet::new(&engine_state);
let output = parse(
&mut working_set,
@ -120,31 +111,15 @@ pub(crate) fn add_menus(
(output, working_set.render())
};
engine_state.merge_delta(delta)?;
let mut temp_stack = Stack::new().capture();
let input = PipelineData::Empty;
menu_eval_results.push(eval_block::<WithoutDebug>(
&engine_state,
&mut temp_stack,
&block,
input,
)?);
}
}
let res = eval_block::<WithoutDebug>(&engine_state, &mut temp_stack, &block, input)?;
let new_engine_state_ref = Arc::new(engine_state);
for res in menu_eval_results.into_iter() {
if let PipelineData::Value(value, None) = res {
for menu in create_menus(&value)? {
line_editor = add_menu(
line_editor,
&menu,
new_engine_state_ref.clone(),
stack,
config.clone(),
)?;
if let PipelineData::Value(value, None) = res {
for menu in create_menus(&value)? {
line_editor =
add_menu(line_editor, &menu, engine_state.clone(), stack, config)?;
}
}
}
}
@ -157,63 +132,47 @@ fn add_menu(
menu: &ParsedMenu,
engine_state: Arc<EngineState>,
stack: &Stack,
config: Arc<Config>,
config: &Config,
) -> Result<Reedline, ShellError> {
let span = menu.menu_type.span();
if let Value::Record { val, .. } = &menu.menu_type {
let layout = extract_value("layout", val, span)?.to_expanded_string("", &config);
let layout = extract_value("layout", val, span)?.to_expanded_string("", config);
match layout.as_str() {
"columnar" => add_columnar_menu(line_editor, menu, engine_state, stack, &config),
"columnar" => add_columnar_menu(line_editor, menu, engine_state, stack, config),
"list" => add_list_menu(line_editor, menu, engine_state, stack, config),
"ide" => add_ide_menu(line_editor, menu, engine_state, stack, config),
"description" => add_description_menu(line_editor, menu, engine_state, stack, config),
_ => Err(ShellError::UnsupportedConfigValue {
expected: "columnar, list, ide or description".to_string(),
value: menu.menu_type.to_abbreviated_string(&config),
value: menu.menu_type.to_abbreviated_string(config),
span: menu.menu_type.span(),
}),
}
} else {
Err(ShellError::UnsupportedConfigValue {
expected: "only record type".to_string(),
value: menu.menu_type.to_abbreviated_string(&config),
value: menu.menu_type.to_abbreviated_string(config),
span: menu.menu_type.span(),
})
}
}
fn get_style(record: &Record, name: &str, span: Span) -> Option<Style> {
extract_value(name, record, span)
.ok()
.map(|text| match text {
Value::String { val, .. } => lookup_ansi_color_style(val),
Value::Record { .. } => color_record_to_nustyle(text),
_ => lookup_ansi_color_style("green"),
})
}
fn set_menu_style<M: MenuBuilder>(mut menu: M, style: &Value) -> M {
let span = style.span();
let Value::Record { val, .. } = &style else {
return menu;
macro_rules! add_style {
// first arm match add!(1,2), add!(2,3) etc
($name:expr, $record: expr, $span:expr, $config: expr, $menu:expr, $f:expr) => {
$menu = match extract_value($name, $record, $span) {
Ok(text) => {
let style = match text {
Value::String { val, .. } => lookup_ansi_color_style(&val),
Value::Record { .. } => color_record_to_nustyle(&text),
_ => lookup_ansi_color_style("green"),
};
$f($menu, style)
}
Err(_) => $menu,
};
};
if let Some(style) = get_style(val, "text", span) {
menu = menu.with_text_style(style);
}
if let Some(style) = get_style(val, "selected_text", span) {
menu = menu.with_selected_text_style(style);
}
if let Some(style) = get_style(val, "description_text", span) {
menu = menu.with_description_text_style(style);
}
if let Some(style) = get_style(val, "match_text", span) {
menu = menu.with_match_text_style(style);
}
if let Some(style) = get_style(val, "selected_match_text", span) {
menu = menu.with_selected_match_text_style(style);
}
menu
}
// Adds a columnar menu to the editor engine
@ -254,7 +213,49 @@ pub(crate) fn add_columnar_menu(
};
}
columnar_menu = set_menu_style(columnar_menu, &menu.style);
let span = menu.style.span();
if let Value::Record { val, .. } = &menu.style {
add_style!(
"text",
val,
span,
config,
columnar_menu,
ColumnarMenu::with_text_style
);
add_style!(
"selected_text",
val,
span,
config,
columnar_menu,
ColumnarMenu::with_selected_text_style
);
add_style!(
"description_text",
val,
span,
config,
columnar_menu,
ColumnarMenu::with_description_text_style
);
add_style!(
"match_text",
val,
span,
config,
columnar_menu,
ColumnarMenu::with_match_text_style
);
add_style!(
"selected_match_text",
val,
span,
config,
columnar_menu,
ColumnarMenu::with_selected_match_text_style
);
}
let marker = menu.marker.to_expanded_string("", config);
columnar_menu = columnar_menu.with_marker(&marker);
@ -294,9 +295,9 @@ pub(crate) fn add_list_menu(
menu: &ParsedMenu,
engine_state: Arc<EngineState>,
stack: &Stack,
config: Arc<Config>,
config: &Config,
) -> Result<Reedline, ShellError> {
let name = menu.name.to_expanded_string("", &config);
let name = menu.name.to_expanded_string("", config);
let mut list_menu = ListMenu::default().with_name(&name);
let span = menu.menu_type.span();
@ -310,9 +311,35 @@ pub(crate) fn add_list_menu(
};
}
list_menu = set_menu_style(list_menu, &menu.style);
let span = menu.style.span();
if let Value::Record { val, .. } = &menu.style {
add_style!(
"text",
val,
span,
config,
list_menu,
ListMenu::with_text_style
);
add_style!(
"selected_text",
val,
span,
config,
list_menu,
ListMenu::with_selected_text_style
);
add_style!(
"description_text",
val,
span,
config,
list_menu,
ListMenu::with_description_text_style
);
}
let marker = menu.marker.to_expanded_string("", &config);
let marker = menu.marker.to_expanded_string("", config);
list_menu = list_menu.with_marker(&marker);
let only_buffer_difference = menu.only_buffer_difference.as_bool()?;
@ -338,7 +365,7 @@ pub(crate) fn add_list_menu(
}
_ => Err(ShellError::UnsupportedConfigValue {
expected: "block or omitted value".to_string(),
value: menu.source.to_abbreviated_string(&config),
value: menu.source.to_abbreviated_string(config),
span: menu.source.span(),
}),
}
@ -350,10 +377,10 @@ pub(crate) fn add_ide_menu(
menu: &ParsedMenu,
engine_state: Arc<EngineState>,
stack: &Stack,
config: Arc<Config>,
config: &Config,
) -> Result<Reedline, ShellError> {
let span = menu.menu_type.span();
let name = menu.name.to_expanded_string("", &config);
let name = menu.name.to_expanded_string("", config);
let mut ide_menu = IdeMenu::default().with_name(&name);
if let Value::Record { val, .. } = &menu.menu_type {
@ -418,7 +445,7 @@ pub(crate) fn add_ide_menu(
} else {
return Err(ShellError::UnsupportedConfigValue {
expected: "bool or record".to_string(),
value: border.to_abbreviated_string(&config),
value: border.to_abbreviated_string(config),
span: border.span(),
});
}
@ -442,7 +469,7 @@ pub(crate) fn add_ide_menu(
_ => {
return Err(ShellError::UnsupportedConfigValue {
expected: "\"left\", \"right\" or \"prefer_right\"".to_string(),
value: description_mode.to_abbreviated_string(&config),
value: description_mode.to_abbreviated_string(config),
span: description_mode.span(),
});
}
@ -491,9 +518,51 @@ pub(crate) fn add_ide_menu(
};
}
ide_menu = set_menu_style(ide_menu, &menu.style);
let span = menu.style.span();
if let Value::Record { val, .. } = &menu.style {
add_style!(
"text",
val,
span,
config,
ide_menu,
IdeMenu::with_text_style
);
add_style!(
"selected_text",
val,
span,
config,
ide_menu,
IdeMenu::with_selected_text_style
);
add_style!(
"description_text",
val,
span,
config,
ide_menu,
IdeMenu::with_description_text_style
);
add_style!(
"match_text",
val,
span,
config,
ide_menu,
IdeMenu::with_match_text_style
);
add_style!(
"selected_match_text",
val,
span,
config,
ide_menu,
IdeMenu::with_selected_match_text_style
);
}
let marker = menu.marker.to_expanded_string("", &config);
let marker = menu.marker.to_expanded_string("", config);
ide_menu = ide_menu.with_marker(&marker);
let only_buffer_difference = menu.only_buffer_difference.as_bool()?;
@ -519,7 +588,7 @@ pub(crate) fn add_ide_menu(
}
_ => Err(ShellError::UnsupportedConfigValue {
expected: "block or omitted value".to_string(),
value: menu.source.to_abbreviated_string(&config),
value: menu.source.to_abbreviated_string(config),
span,
}),
}
@ -531,9 +600,9 @@ pub(crate) fn add_description_menu(
menu: &ParsedMenu,
engine_state: Arc<EngineState>,
stack: &Stack,
config: Arc<Config>,
config: &Config,
) -> Result<Reedline, ShellError> {
let name = menu.name.to_expanded_string("", &config);
let name = menu.name.to_expanded_string("", config);
let mut description_menu = DescriptionMenu::default().with_name(&name);
let span = menu.menu_type.span();
@ -579,9 +648,35 @@ pub(crate) fn add_description_menu(
};
}
description_menu = set_menu_style(description_menu, &menu.style);
let span = menu.style.span();
if let Value::Record { val, .. } = &menu.style {
add_style!(
"text",
val,
span,
config,
description_menu,
DescriptionMenu::with_text_style
);
add_style!(
"selected_text",
val,
span,
config,
description_menu,
DescriptionMenu::with_selected_text_style
);
add_style!(
"description_text",
val,
span,
config,
description_menu,
DescriptionMenu::with_description_text_style
);
}
let marker = menu.marker.to_expanded_string("", &config);
let marker = menu.marker.to_expanded_string("", config);
description_menu = description_menu.with_marker(&marker);
let only_buffer_difference = menu.only_buffer_difference.as_bool()?;
@ -590,7 +685,7 @@ pub(crate) fn add_description_menu(
let span = menu.source.span();
match &menu.source {
Value::Nothing { .. } => {
let completer = Box::new(NuHelpCompleter::new(engine_state, config));
let completer = Box::new(NuHelpCompleter::new(engine_state));
Ok(line_editor.with_menu(ReedlineMenu::WithCompleter {
menu: Box::new(description_menu),
completer,
@ -611,7 +706,7 @@ pub(crate) fn add_description_menu(
}
_ => Err(ShellError::UnsupportedConfigValue {
expected: "closure or omitted value".to_string(),
value: menu.source.to_abbreviated_string(&config),
value: menu.source.to_abbreviated_string(config),
span: menu.source.span(),
}),
}

883
crates/nu-cli/src/repl.rs
View File

File diff suppressed because it is too large Load Diff

226
crates/nu-cli/src/syntax_highlight.rs
View File

@ -4,9 +4,9 @@ use nu_color_config::{get_matching_brackets_style, get_shape_color};
use nu_engine::env;
use nu_parser::{flatten_block, parse, FlatShape};
use nu_protocol::{
ast::{Block, Expr, Expression, PipelineRedirection, RecordItem},
ast::{Argument, Block, Expr, Expression, PipelineRedirection, RecordItem},
engine::{EngineState, Stack, StateWorkingSet},
Span,
Config, Span,
};
use reedline::{Highlighter, StyledText};
use std::sync::Arc;
@ -14,14 +14,15 @@ use std::sync::Arc;
pub struct NuHighlighter {
pub engine_state: Arc<EngineState>,
pub stack: Arc<Stack>,
pub config: Config,
}
impl Highlighter for NuHighlighter {
fn highlight(&self, line: &str, _cursor: usize) -> StyledText {
trace!("highlighting: {}", line);
let config = self.stack.get_config(&self.engine_state);
let highlight_resolved_externals = config.highlight_resolved_externals;
let highlight_resolved_externals =
self.engine_state.get_config().highlight_resolved_externals;
let mut working_set = StateWorkingSet::new(&self.engine_state);
let block = parse(&mut working_set, None, line.as_bytes(), false);
let (shapes, global_span_offset) = {
@ -36,7 +37,6 @@ impl Highlighter for NuHighlighter {
let str_word = String::from_utf8_lossy(str_contents).to_string();
let paths = env::path_str(&self.engine_state, &self.stack, *span).ok();
#[allow(deprecated)]
let res = if let Ok(cwd) =
env::current_dir_str(&self.engine_state, &self.stack)
{
@ -86,8 +86,29 @@ impl Highlighter for NuHighlighter {
[(shape.0.start - global_span_offset)..(shape.0.end - global_span_offset)]
.to_string();
macro_rules! add_colored_token_with_bracket_highlight {
($shape:expr, $span:expr, $text:expr) => {{
let spans = split_span_by_highlight_positions(
line,
$span,
&matching_brackets_pos,
global_span_offset,
);
spans.iter().for_each(|(part, highlight)| {
let start = part.start - $span.start;
let end = part.end - $span.start;
let text = (&next_token[start..end]).to_string();
let mut style = get_shape_color($shape.to_string(), &self.config);
if *highlight {
style = get_matching_brackets_style(style, &self.config);
}
output.push((style, text));
});
}};
}
let mut add_colored_token = |shape: &FlatShape, text: String| {
output.push((get_shape_color(shape.as_str(), &config), text));
output.push((get_shape_color(shape.to_string(), &self.config), text));
};
match shape.1 {
@ -107,37 +128,27 @@ impl Highlighter for NuHighlighter {
FlatShape::Operator => add_colored_token(&shape.1, next_token),
FlatShape::Signature => add_colored_token(&shape.1, next_token),
FlatShape::String => add_colored_token(&shape.1, next_token),
FlatShape::RawString => add_colored_token(&shape.1, next_token),
FlatShape::StringInterpolation => add_colored_token(&shape.1, next_token),
FlatShape::DateTime => add_colored_token(&shape.1, next_token),
FlatShape::List
| FlatShape::Table
| FlatShape::Record
| FlatShape::Block
| FlatShape::Closure => {
let span = shape.0;
let shape = &shape.1;
let spans = split_span_by_highlight_positions(
line,
span,
&matching_brackets_pos,
global_span_offset,
);
for (part, highlight) in spans {
let start = part.start - span.start;
let end = part.end - span.start;
let text = next_token[start..end].to_string();
let mut style = get_shape_color(shape.as_str(), &config);
if highlight {
style = get_matching_brackets_style(style, &config);
}
output.push((style, text));
}
FlatShape::List => {
add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
}
FlatShape::Table => {
add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
}
FlatShape::Record => {
add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
}
FlatShape::Block => {
add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
}
FlatShape::Closure => {
add_colored_token_with_bracket_highlight!(shape.1, shape.0, next_token)
}
FlatShape::Filepath => add_colored_token(&shape.1, next_token),
FlatShape::Directory => add_colored_token(&shape.1, next_token),
FlatShape::GlobInterpolation => add_colored_token(&shape.1, next_token),
FlatShape::GlobPattern => add_colored_token(&shape.1, next_token),
FlatShape::Variable(_) | FlatShape::VarDecl(_) => {
add_colored_token(&shape.1, next_token)
@ -299,6 +310,20 @@ fn find_matching_block_end_in_expr(
global_span_offset: usize,
global_cursor_offset: usize,
) -> Option<usize> {
macro_rules! find_in_expr_or_continue {
($inner_expr:ident) => {
if let Some(pos) = find_matching_block_end_in_expr(
line,
working_set,
$inner_expr,
global_span_offset,
global_cursor_offset,
) {
return Some(pos);
}
};
}
if expression.span.contains(global_cursor_offset) && expression.span.start >= global_span_offset
{
let expr_first = expression.span.start;
@ -328,7 +353,6 @@ fn find_matching_block_end_in_expr(
Expr::Directory(_, _) => None,
Expr::GlobPattern(_, _) => None,
Expr::String(_) => None,
Expr::RawString(_) => None,
Expr::CellPath(_) => None,
Expr::ImportPattern(_) => None,
Expr::Overlay(_) => None,
@ -346,19 +370,15 @@ fn find_matching_block_end_in_expr(
Some(expr_last)
} else {
// cursor is inside table
table
.columns
.iter()
.chain(table.rows.iter().flat_map(AsRef::as_ref))
.find_map(|expr| {
find_matching_block_end_in_expr(
line,
working_set,
expr,
global_span_offset,
global_cursor_offset,
)
})
for inner_expr in table.columns.as_ref() {
find_in_expr_or_continue!(inner_expr);
}
for row in table.rows.as_ref() {
for inner_expr in row.as_ref() {
find_in_expr_or_continue!(inner_expr);
}
}
None
}
}
@ -371,45 +391,36 @@ fn find_matching_block_end_in_expr(
Some(expr_last)
} else {
// cursor is inside record
exprs.iter().find_map(|expr| match expr {
RecordItem::Pair(k, v) => find_matching_block_end_in_expr(
line,
working_set,
k,
global_span_offset,
global_cursor_offset,
)
.or_else(|| {
find_matching_block_end_in_expr(
line,
working_set,
v,
global_span_offset,
global_cursor_offset,
)
}),
RecordItem::Spread(_, record) => find_matching_block_end_in_expr(
line,
working_set,
record,
global_span_offset,
global_cursor_offset,
),
})
for expr in exprs {
match expr {
RecordItem::Pair(k, v) => {
find_in_expr_or_continue!(k);
find_in_expr_or_continue!(v);
}
RecordItem::Spread(_, record) => {
find_in_expr_or_continue!(record);
}
}
}
None
}
}
Expr::Call(call) => call.arguments.iter().find_map(|arg| {
arg.expr().and_then(|expr| {
find_matching_block_end_in_expr(
line,
working_set,
expr,
global_span_offset,
global_cursor_offset,
)
})
}),
Expr::Call(call) => {
for arg in &call.arguments {
let opt_expr = match arg {
Argument::Named((_, _, opt_expr)) => opt_expr.as_ref(),
Argument::Positional(inner_expr) => Some(inner_expr),
Argument::Unknown(inner_expr) => Some(inner_expr),
Argument::Spread(inner_expr) => Some(inner_expr),
};
if let Some(inner_expr) = opt_expr {
find_in_expr_or_continue!(inner_expr);
}
}
None
}
Expr::FullCellPath(b) => find_matching_block_end_in_expr(
line,
@ -419,23 +430,12 @@ fn find_matching_block_end_in_expr(
global_cursor_offset,
),
Expr::BinaryOp(lhs, op, rhs) => [lhs, op, rhs].into_iter().find_map(|expr| {
find_matching_block_end_in_expr(
line,
working_set,
expr,
global_span_offset,
global_cursor_offset,
)
}),
Expr::Collect(_, expr) => find_matching_block_end_in_expr(
line,
working_set,
expr,
global_span_offset,
global_cursor_offset,
),
Expr::BinaryOp(lhs, op, rhs) => {
find_in_expr_or_continue!(lhs);
find_in_expr_or_continue!(op);
find_in_expr_or_continue!(rhs);
None
}
Expr::Block(block_id)
| Expr::Closure(block_id)
@ -460,16 +460,11 @@ fn find_matching_block_end_in_expr(
}
}
Expr::StringInterpolation(exprs) | Expr::GlobInterpolation(exprs, _) => {
exprs.iter().find_map(|expr| {
find_matching_block_end_in_expr(
line,
working_set,
expr,
global_span_offset,
global_cursor_offset,
)
})
Expr::StringInterpolation(inner_expr) => {
for inner_expr in inner_expr {
find_in_expr_or_continue!(inner_expr);
}
None
}
Expr::List(list) => {
@ -480,15 +475,12 @@ fn find_matching_block_end_in_expr(
// cursor is at list start
Some(expr_last)
} else {
list.iter().find_map(|item| {
find_matching_block_end_in_expr(
line,
working_set,
item.expr(),
global_span_offset,
global_cursor_offset,
)
})
// cursor is inside list
for item in list {
let expr = item.expr();
find_in_expr_or_continue!(expr);
}
None
}
}
};

179
crates/nu-cli/src/util.rs
View File

@ -4,11 +4,11 @@ use nu_parser::{escape_quote_string, lex, parse, unescape_unquote_string, Token,
use nu_protocol::{
debugger::WithoutDebug,
engine::{EngineState, Stack, StateWorkingSet},
report_error, report_error_new, PipelineData, ShellError, Span, Value,
print_if_stream, report_error, report_error_new, PipelineData, ShellError, Span, Value,
};
#[cfg(windows)]
use nu_utils::enable_vt_processing;
use nu_utils::perf;
use nu_utils::utils::perf;
use std::path::Path;
// This will collect environment variables from std::env and adds them to a stack.
@ -39,8 +39,9 @@ fn gather_env_vars(
init_cwd: &Path,
) {
fn report_capture_error(engine_state: &EngineState, env_str: &str, msg: &str) {
report_error_new(
engine_state,
let working_set = StateWorkingSet::new(engine_state);
report_error(
&working_set,
&ShellError::GenericError {
error: format!("Environment variable was not captured: {env_str}"),
msg: "".into(),
@ -70,8 +71,9 @@ fn gather_env_vars(
}
None => {
// Could not capture current working directory
report_error_new(
engine_state,
let working_set = StateWorkingSet::new(engine_state);
report_error(
&working_set,
&ShellError::GenericError {
error: "Current directory is not a valid utf-8 path".into(),
msg: "".into(),
@ -206,45 +208,9 @@ pub fn eval_source(
fname: &str,
input: PipelineData,
allow_return: bool,
) -> i32 {
) -> bool {
let start_time = std::time::Instant::now();
let exit_code = match evaluate_source(engine_state, stack, source, fname, input, allow_return) {
Ok(code) => code.unwrap_or(0),
Err(err) => {
report_error_new(engine_state, &err);
1
}
};
stack.add_env_var(
"LAST_EXIT_CODE".to_string(),
Value::int(exit_code.into(), Span::unknown()),
);
// reset vt processing, aka ansi because illbehaved externals can break it
#[cfg(windows)]
{
let _ = enable_vt_processing();
}
perf!(
&format!("eval_source {}", &fname),
start_time,
engine_state.get_config().use_ansi_coloring
);
exit_code
}
fn evaluate_source(
engine_state: &mut EngineState,
stack: &mut Stack,
source: &[u8],
fname: &str,
input: PipelineData,
allow_return: bool,
) -> Result<Option<i32>, ShellError> {
let (block, delta) = {
let mut working_set = StateWorkingSet::new(engine_state);
let output = parse(
@ -258,45 +224,104 @@ fn evaluate_source(
}
if let Some(err) = working_set.parse_errors.first() {
set_last_exit_code(stack, 1);
report_error(&working_set, err);
return Ok(Some(1));
}
if let Some(err) = working_set.compile_errors.first() {
report_error(&working_set, err);
// Not a fatal error, for now
return false;
}
(output, working_set.render())
};
engine_state.merge_delta(delta)?;
if let Err(err) = engine_state.merge_delta(delta) {
set_last_exit_code(stack, 1);
report_error_new(engine_state, &err);
return false;
}
let pipeline = if allow_return {
let b = if allow_return {
eval_block_with_early_return::<WithoutDebug>(engine_state, stack, &block, input)
} else {
eval_block::<WithoutDebug>(engine_state, stack, &block, input)
}?;
let status = if let PipelineData::ByteStream(..) = pipeline {
pipeline.print(engine_state, stack, false, false)?
} else {
if let Some(hook) = engine_state.get_config().hooks.display_output.clone() {
let pipeline = eval_hook(
engine_state,
stack,
Some(pipeline),
vec![],
&hook,
"display_output",
)?;
pipeline.print(engine_state, stack, false, false)
} else {
pipeline.print(engine_state, stack, true, false)
}?
};
Ok(status.map(|status| status.code()))
match b {
Ok(pipeline_data) => {
let config = engine_state.get_config();
let result;
if let PipelineData::ExternalStream {
stdout: stream,
stderr: stderr_stream,
exit_code,
..
} = pipeline_data
{
result = print_if_stream(stream, stderr_stream, false, exit_code);
} else if let Some(hook) = config.hooks.display_output.clone() {
match eval_hook(
engine_state,
stack,
Some(pipeline_data),
vec![],
&hook,
"display_output",
) {
Err(err) => {
result = Err(err);
}
Ok(val) => {
result = val.print(engine_state, stack, false, false);
}
}
} else {
result = pipeline_data.print(engine_state, stack, true, false);
}
match result {
Err(err) => {
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &err);
return false;
}
Ok(exit_code) => {
set_last_exit_code(stack, exit_code);
}
}
// reset vt processing, aka ansi because illbehaved externals can break it
#[cfg(windows)]
{
let _ = enable_vt_processing();
}
}
Err(err) => {
set_last_exit_code(stack, 1);
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &err);
return false;
}
}
perf(
&format!("eval_source {}", &fname),
start_time,
file!(),
line!(),
column!(),
engine_state.get_config().use_ansi_coloring,
);
true
}
fn set_last_exit_code(stack: &mut Stack, exit_code: i64) {
stack.add_env_var(
"LAST_EXIT_CODE".to_string(),
Value::int(exit_code, Span::unknown()),
);
}
#[cfg(test)]
@ -321,10 +346,16 @@ mod test {
let env = engine_state.render_env_vars();
assert!(matches!(env.get("FOO"), Some(&Value::String { val, .. }) if val == "foo"));
assert!(matches!(env.get("SYMBOLS"), Some(&Value::String { val, .. }) if val == symbols));
assert!(matches!(env.get(symbols), Some(&Value::String { val, .. }) if val == "symbols"));
assert!(env.contains_key("PWD"));
assert!(
matches!(env.get(&"FOO".to_string()), Some(&Value::String { val, .. }) if val == "foo")
);
assert!(
matches!(env.get(&"SYMBOLS".to_string()), Some(&Value::String { val, .. }) if val == symbols)
);
assert!(
matches!(env.get(&symbols.to_string()), Some(&Value::String { val, .. }) if val == "symbols")
);
assert!(env.get(&"PWD".to_string()).is_some());
assert_eq!(env.len(), 4);
}
}

7
crates/nu-cli/tests/commands/keybindings_list.rs
View File

@ -1,7 +0,0 @@
use nu_test_support::nu;
#[test]
fn not_empty() {
let result = nu!("keybindings list | is-not-empty");
assert_eq!(result.out, "true");
}

2
crates/nu-cli/tests/commands/mod.rs
View File

@ -1,2 +0,0 @@
mod keybindings_list;
mod nu_highlight;

7
crates/nu-cli/tests/commands/nu_highlight.rs
View File

@ -1,7 +0,0 @@
use nu_test_support::nu;
#[test]
fn nu_highlight_not_expr() {
let actual = nu!("'not false' | nu-highlight | ansi strip");
assert_eq!(actual.out, "not false");
}

494
crates/nu-cli/tests/completions/mod.rs → crates/nu-cli/tests/completions.rs
View File

File diff suppressed because it is too large Load Diff

2
crates/nu-cli/tests/main.rs
View File

@ -1,2 +0,0 @@
mod commands;
mod completions;

96
crates/nu-cli/tests/completions/support/completions_helpers.rs → crates/nu-cli/tests/support/completions_helpers.rs
View File

@ -1,34 +1,39 @@
use nu_engine::eval_block;
use nu_parser::parse;
use nu_path::{AbsolutePathBuf, PathBuf};
use nu_protocol::{
debugger::WithoutDebug,
engine::{EngineState, Stack, StateWorkingSet},
PipelineData, ShellError, Span, Value,
eval_const::create_nu_constant,
PipelineData, ShellError, Span, Value, NU_VARIABLE_ID,
};
use nu_test_support::fs;
use reedline::Suggestion;
use std::path::MAIN_SEPARATOR;
use std::path::PathBuf;
const SEP: char = std::path::MAIN_SEPARATOR;
fn create_default_context() -> EngineState {
nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
}
// creates a new engine with the current path into the completions fixtures folder
pub fn new_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
pub fn new_engine() -> (PathBuf, String, EngineState, Stack) {
// Target folder inside assets
let dir = fs::fixtures().join("completions");
let dir_str = dir
let mut dir_str = dir
.clone()
.into_os_string()
.into_string()
.unwrap_or_default();
dir_str.push(SEP);
// Create a new engine with default context
let mut engine_state = create_default_context();
// Add $nu
engine_state.generate_nu_constant();
let nu_const =
create_nu_constant(&engine_state, Span::test_data()).expect("Failed creating $nu");
engine_state.set_variable_const_val(NU_VARIABLE_ID, nu_const);
// New stack
let mut stack = Stack::new();
@ -69,60 +74,15 @@ pub fn new_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
(dir, dir_str, engine_state, stack)
}
// creates a new engine with the current path into the completions fixtures folder
pub fn new_dotnu_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
// Target folder inside assets
let dir = fs::fixtures().join("dotnu_completions");
let dir_str = dir
.clone()
.into_os_string()
.into_string()
.unwrap_or_default();
let dir_span = nu_protocol::Span::new(0, dir_str.len());
// Create a new engine with default context
let mut engine_state = create_default_context();
// Add $nu
engine_state.generate_nu_constant();
// New stack
let mut stack = Stack::new();
// Add pwd as env var
stack.add_env_var("PWD".to_string(), Value::string(dir_str.clone(), dir_span));
stack.add_env_var(
"TEST".to_string(),
Value::string("NUSHELL".to_string(), dir_span),
);
stack.add_env_var(
"NU_LIB_DIRS".to_string(),
Value::List {
vals: vec![
Value::string(file(dir.join("lib-dir1")), dir_span),
Value::string(file(dir.join("lib-dir2")), dir_span),
Value::string(file(dir.join("lib-dir3")), dir_span),
],
internal_span: dir_span,
},
);
// Merge environment into the permanent state
let merge_result = engine_state.merge_env(&mut stack, &dir);
assert!(merge_result.is_ok());
(dir, dir_str, engine_state, stack)
}
pub fn new_quote_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
pub fn new_quote_engine() -> (PathBuf, String, EngineState, Stack) {
// Target folder inside assets
let dir = fs::fixtures().join("quoted_completions");
let dir_str = dir
let mut dir_str = dir
.clone()
.into_os_string()
.into_string()
.unwrap_or_default();
dir_str.push(SEP);
// Create a new engine with default context
let mut engine_state = create_default_context();
@ -150,14 +110,15 @@ pub fn new_quote_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
(dir, dir_str, engine_state, stack)
}
pub fn new_partial_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
pub fn new_partial_engine() -> (PathBuf, String, EngineState, Stack) {
// Target folder inside assets
let dir = fs::fixtures().join("partial_completions");
let dir_str = dir
let mut dir_str = dir
.clone()
.into_os_string()
.into_string()
.unwrap_or_default();
dir_str.push(SEP);
// Create a new engine with default context
let mut engine_state = create_default_context();
@ -186,7 +147,7 @@ pub fn new_partial_engine() -> (AbsolutePathBuf, String, EngineState, Stack) {
}
// match a list of suggestions with the expected values
pub fn match_suggestions(expected: &Vec<String>, suggestions: &Vec<Suggestion>) {
pub fn match_suggestions(expected: Vec<String>, suggestions: Vec<Suggestion>) {
let expected_len = expected.len();
let suggestions_len = suggestions.len();
if expected_len != suggestions_len {
@ -196,25 +157,22 @@ pub fn match_suggestions(expected: &Vec<String>, suggestions: &Vec<Suggestion>)
Expected: {expected:#?}\n"
)
}
let suggestoins_str = suggestions
.iter()
.map(|it| it.value.clone())
.collect::<Vec<_>>();
assert_eq!(expected, &suggestoins_str);
expected.iter().zip(suggestions).for_each(|it| {
assert_eq!(it.0, &it.1.value);
});
}
// append the separator to the converted path
pub fn folder(path: impl Into<PathBuf>) -> String {
pub fn folder(path: PathBuf) -> String {
let mut converted_path = file(path);
converted_path.push(MAIN_SEPARATOR);
converted_path.push(SEP);
converted_path
}
// convert a given path to string
pub fn file(path: impl Into<PathBuf>) -> String {
path.into().into_os_string().into_string().unwrap()
pub fn file(path: PathBuf) -> String {
path.into_os_string().into_string().unwrap_or_default()
}
// merge_input executes the given input into the engine
@ -223,7 +181,7 @@ pub fn merge_input(
input: &[u8],
engine_state: &mut EngineState,
stack: &mut Stack,
dir: AbsolutePathBuf,
dir: PathBuf,
) -> Result<(), ShellError> {
let (block, delta) = {
let mut working_set = StateWorkingSet::new(engine_state);

0
crates/nu-cli/tests/completions/support/mod.rs → crates/nu-cli/tests/support/mod.rs
View File

12
crates/nu-cmd-base/Cargo.toml
View File

@ -5,17 +5,17 @@ edition = "2021"
license = "MIT"
name = "nu-cmd-base"
repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-base"
version = "0.96.2"
version = "0.92.3"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
nu-engine = { path = "../nu-engine", version = "0.96.2" }
nu-parser = { path = "../nu-parser", version = "0.96.2" }
nu-path = { path = "../nu-path", version = "0.96.2" }
nu-protocol = { path = "../nu-protocol", version = "0.96.2" }
nu-engine = { path = "../nu-engine", version = "0.92.3" }
nu-parser = { path = "../nu-parser", version = "0.92.3" }
nu-path = { path = "../nu-path", version = "0.92.3" }
nu-protocol = { path = "../nu-protocol", version = "0.92.3" }
indexmap = { workspace = true }
miette = { workspace = true }
[dev-dependencies]
[dev-dependencies]

5
crates/nu-cmd-base/README.md
View File

@ -1,5 +0,0 @@
Utilities used by the different `nu-command`/`nu-cmd-*` crates, should not contain any full `Command` implementations.
## Internal Nushell crate
This crate implements components of Nushell and is not designed to support plugin authors or other users directly.

2
crates/nu-cmd-base/src/hook.rs
View File

@ -194,7 +194,7 @@ pub fn eval_hook(
let Some(follow) = val.get("code") else {
return Err(ShellError::CantFindColumn {
col_name: "code".into(),
span: Some(span),
span,
src_span: span,
});
};

10
crates/nu-cmd-base/src/input_handler.rs
View File

@ -1,5 +1,5 @@
use nu_protocol::{ast::CellPath, PipelineData, ShellError, Signals, Span, Value};
use std::sync::Arc;
use nu_protocol::{ast::CellPath, PipelineData, ShellError, Span, Value};
use std::sync::{atomic::AtomicBool, Arc};
pub trait CmdArgument {
fn take_cell_paths(&mut self) -> Option<Vec<CellPath>>;
@ -40,7 +40,7 @@ pub fn operate<C, A>(
mut arg: A,
input: PipelineData,
span: Span,
signals: &Signals,
ctrlc: Option<Arc<AtomicBool>>,
) -> Result<PipelineData, ShellError>
where
A: CmdArgument + Send + Sync + 'static,
@ -55,7 +55,7 @@ where
_ => cmd(&v, &arg, span),
}
},
signals,
ctrlc,
),
Some(column_paths) => {
let arg = Arc::new(arg);
@ -79,7 +79,7 @@ where
}
v
},
signals,
ctrlc,
)
}
}

1
crates/nu-cmd-base/src/lib.rs
View File

@ -1,4 +1,3 @@
#![doc = include_str!("../README.md")]
pub mod formats;
pub mod hook;
pub mod input_handler;

36
crates/nu-cmd-base/src/util.rs
View File

@ -1,33 +1,27 @@
use nu_path::AbsolutePathBuf;
use nu_protocol::{
engine::{EngineState, Stack},
Range, ShellError, Span, Value,
engine::{EngineState, Stack, StateWorkingSet},
report_error, Range, ShellError, Span, Value,
};
use std::ops::Bound;
use std::{ops::Bound, path::PathBuf};
pub fn get_init_cwd() -> AbsolutePathBuf {
std::env::current_dir()
.ok()
.and_then(|path| AbsolutePathBuf::try_from(path).ok())
.or_else(|| {
std::env::var("PWD")
.ok()
.and_then(|path| AbsolutePathBuf::try_from(path).ok())
})
.or_else(nu_path::home_dir)
.expect("Failed to get current working directory")
pub fn get_init_cwd() -> PathBuf {
std::env::current_dir().unwrap_or_else(|_| {
std::env::var("PWD")
.map(Into::into)
.unwrap_or_else(|_| nu_path::home_dir().unwrap_or_default())
})
}
pub fn get_guaranteed_cwd(engine_state: &EngineState, stack: &Stack) -> AbsolutePathBuf {
engine_state
.cwd(Some(stack))
.ok()
.unwrap_or_else(get_init_cwd)
pub fn get_guaranteed_cwd(engine_state: &EngineState, stack: &Stack) -> PathBuf {
nu_engine::env::current_dir(engine_state, stack).unwrap_or_else(|e| {
let working_set = StateWorkingSet::new(engine_state);
report_error(&working_set, &e);
crate::util::get_init_cwd()
})
}
type MakeRangeError = fn(&str, Span) -> ShellError;
/// Returns a inclusive pair of boundary in given `range`.
pub fn process_range(range: &Range) -> Result<(isize, isize), MakeRangeError> {
match range {
Range::IntRange(range) => {

75
crates/nu-cmd-dataframe/Cargo.toml Normal file
View File

@ -0,0 +1,75 @@
[package]
authors = ["The Nushell Project Developers"]
description = "Nushell's dataframe commands based on polars."
edition = "2021"
license = "MIT"
name = "nu-cmd-dataframe"
repository = "https://github.com/nushell/nushell/tree/main/crates/nu-cmd-dataframe"
version = "0.92.3"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
bench = false
[dependencies]
nu-engine = { path = "../nu-engine", version = "0.92.3" }
nu-parser = { path = "../nu-parser", version = "0.92.3" }
nu-protocol = { path = "../nu-protocol", version = "0.92.3" }
# Potential dependencies for extras
chrono = { workspace = true, features = ["std", "unstable-locales"], default-features = false }
chrono-tz = { workspace = true }
fancy-regex = { workspace = true }
indexmap = { workspace = true }
num = { version = "0.4", optional = true }
serde = { workspace = true, features = ["derive"] }
# keep sqlparser at 0.39.0 until we can update polars
sqlparser = { version = "0.45", optional = true }
polars-io = { version = "0.39", features = ["avro"], optional = true }
polars-arrow = { version = "0.39", optional = true }
polars-ops = { version = "0.39", optional = true }
polars-plan = { version = "0.39", features = ["regex"], optional = true }
polars-utils = { version = "0.39", optional = true }
[dependencies.polars]
features = [
"arg_where",
"checked_arithmetic",
"concat_str",
"cross_join",
"csv",
"cum_agg",
"dtype-categorical",
"dtype-datetime",
"dtype-struct",
"dtype-i8",
"dtype-i16",
"dtype-u8",
"dtype-u16",
"dynamic_group_by",
"ipc",
"is_in",
"json",
"lazy",
"object",
"parquet",
"random",
"rolling_window",
"rows",
"serde",
"serde-lazy",
"strings",
"temporal",
"to_dummies",
]
default-features = false
optional = true
version = "0.39"
[features]
dataframe = ["num", "polars", "polars-io", "polars-arrow", "polars-ops", "polars-plan", "polars-utils", "sqlparser"]
default = []
[dev-dependencies]
nu-cmd-lang = { path = "../nu-cmd-lang", version = "0.92.3" }

0
crates/nu-derive-value/LICENSE → crates/nu-cmd-dataframe/LICENSE
View File

12
crates/nu-cmd-dataframe/src/dataframe/README.md Normal file
View File

@ -0,0 +1,12 @@
# Dataframe
This dataframe directory holds all of the definitions of the dataframe data structures and commands.
There are three sections of commands:
* [eager](./eager)
* [series](./series)
* [values](./values)
For more details see the
[Nushell book section on dataframes](https://www.nushell.sh/book/dataframes.html)

134
crates/nu-cmd-dataframe/src/dataframe/eager/append.rs Normal file
View File

@ -0,0 +1,134 @@
use crate::dataframe::values::{Axis, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct AppendDF;
impl Command for AppendDF {
fn name(&self) -> &str {
"dfr append"
}
fn usage(&self) -> &str {
"Appends a new dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("other", SyntaxShape::Any, "dataframe to be appended")
.switch("col", "appends in col orientation", Some('c'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Appends a dataframe as new columns",
example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
$a | dfr append $a"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
Column::new(
"a_x".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b_x".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Appends a dataframe merging at the end of columns",
example: r#"let a = ([[a b]; [1 2] [3 4]] | dfr into-df);
$a | dfr append $a --col"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![
Value::test_int(1),
Value::test_int(3),
Value::test_int(1),
Value::test_int(3),
],
),
Column::new(
"b".to_string(),
vec![
Value::test_int(2),
Value::test_int(4),
Value::test_int(2),
Value::test_int(4),
],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let other: Value = call.req(engine_state, stack, 0)?;
let axis = if call.has_flag(engine_state, stack, "col")? {
Axis::Column
} else {
Axis::Row
};
let df_other = NuDataFrame::try_from_value(other)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
df.append_df(&df_other, axis, call.head)
.map(|df| PipelineData::Value(NuDataFrame::into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(AppendDF {})])
}
}

195
crates/nu-cmd-dataframe/src/dataframe/eager/cast.rs Normal file
View File

@ -0,0 +1,195 @@
use crate::dataframe::values::{str_to_dtype, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
use polars::prelude::*;
#[derive(Clone)]
pub struct CastDF;
impl Command for CastDF {
fn name(&self) -> &str {
"dfr cast"
}
fn usage(&self) -> &str {
"Cast a column to a different dtype."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.required(
"dtype",
SyntaxShape::String,
"The dtype to cast the column to",
)
.optional(
"column",
SyntaxShape::String,
"The column to cast. Required when used with a dataframe.",
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Cast a column in a dataframe to a different dtype",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr cast u8 a | dfr schema",
result: Some(Value::record(
record! {
"a" => Value::string("u8", Span::test_data()),
"b" => Value::string("i64", Span::test_data()),
},
Span::test_data(),
)),
},
Example {
description: "Cast a column in a lazy dataframe to a different dtype",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr into-lazy | dfr cast u8 a | dfr schema",
result: Some(Value::record(
record! {
"a" => Value::string("u8", Span::test_data()),
"b" => Value::string("i64", Span::test_data()),
},
Span::test_data(),
)),
},
Example {
description: "Cast a column in a expression to a different dtype",
example: r#"[[a b]; [1 2] [1 4]] | dfr into-df | dfr group-by a | dfr agg [ (dfr col b | dfr cast u8 | dfr min | dfr as "b_min") ] | dfr schema"#,
result: None
}
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head);
if NuLazyFrame::can_downcast(&value) {
let (dtype, column_nm) = df_args(engine_state, stack, call)?;
let df = NuLazyFrame::try_from_value(value)?;
command_lazy(call, column_nm, dtype, df)
} else if NuDataFrame::can_downcast(&value) {
let (dtype, column_nm) = df_args(engine_state, stack, call)?;
let df = NuDataFrame::try_from_value(value)?;
command_eager(call, column_nm, dtype, df)
} else {
let dtype: String = call.req(engine_state, stack, 0)?;
let dtype = str_to_dtype(&dtype, call.head)?;
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().cast(dtype).into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
}
fn df_args(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<(DataType, String), ShellError> {
let dtype = dtype_arg(engine_state, stack, call)?;
let column_nm: String =
call.opt(engine_state, stack, 1)?
.ok_or(ShellError::MissingParameter {
param_name: "column_name".into(),
span: call.head,
})?;
Ok((dtype, column_nm))
}
fn dtype_arg(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<DataType, ShellError> {
let dtype: String = call.req(engine_state, stack, 0)?;
str_to_dtype(&dtype, call.head)
}
fn command_lazy(
call: &Call,
column_nm: String,
dtype: DataType,
lazy: NuLazyFrame,
) -> Result<PipelineData, ShellError> {
let column = col(&column_nm).cast(dtype);
let lazy = lazy.into_polars().with_columns(&[column]);
let lazy = NuLazyFrame::new(false, lazy);
Ok(PipelineData::Value(
NuLazyFrame::into_value(lazy, call.head)?,
None,
))
}
fn command_eager(
call: &Call,
column_nm: String,
dtype: DataType,
nu_df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let mut df = nu_df.df;
let column = df
.column(&column_nm)
.map_err(|e| ShellError::GenericError {
error: format!("{e}"),
msg: "".into(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let casted = column.cast(&dtype).map_err(|e| ShellError::GenericError {
error: format!("{e}"),
msg: "".into(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let _ = df
.with_column(casted)
.map_err(|e| ShellError::GenericError {
error: format!("{e}"),
msg: "".into(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let df = NuDataFrame::new(false, df);
Ok(PipelineData::Value(df.into_value(call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(CastDF {})])
}
}

73
crates/nu-cmd-dataframe/src/dataframe/eager/columns.rs Normal file
View File

@ -0,0 +1,73 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ColumnsDF;
impl Command for ColumnsDF {
fn name(&self) -> &str {
"dfr columns"
}
fn usage(&self) -> &str {
"Show dataframe columns."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(Type::Custom("dataframe".into()), Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Dataframe columns",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr columns",
result: Some(Value::list(
vec![Value::test_string("a"), Value::test_string("b")],
Span::test_data(),
)),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let names: Vec<Value> = df
.as_ref()
.get_column_names()
.iter()
.map(|v| Value::string(*v, call.head))
.collect();
let names = Value::list(names, call.head);
Ok(PipelineData::Value(names, None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ColumnsDF {})])
}
}

115
crates/nu-cmd-dataframe/src/dataframe/eager/drop.rs Normal file
View File

@ -0,0 +1,115 @@
use crate::dataframe::values::{utils::convert_columns, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct DropDF;
impl Command for DropDF {
fn name(&self) -> &str {
"dfr drop"
}
fn usage(&self) -> &str {
"Creates a new dataframe by dropping the selected columns."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.rest("rest", SyntaxShape::Any, "column names to be dropped")
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "drop column a",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr drop a",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let columns: Vec<Value> = call.rest(engine_state, stack, 0)?;
let (col_string, col_span) = convert_columns(columns, call.head)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let new_df = col_string
.first()
.ok_or_else(|| ShellError::GenericError {
error: "Empty names list".into(),
msg: "No column names were found".into(),
span: Some(col_span),
help: None,
inner: vec![],
})
.and_then(|col| {
df.as_ref()
.drop(&col.item)
.map_err(|e| ShellError::GenericError {
error: "Error dropping column".into(),
msg: e.to_string(),
span: Some(col.span),
help: None,
inner: vec![],
})
})?;
// If there are more columns in the drop selection list, these
// are added from the resulting dataframe
col_string
.iter()
.skip(1)
.try_fold(new_df, |new_df, col| {
new_df
.drop(&col.item)
.map_err(|e| ShellError::GenericError {
error: "Error dropping column".into(),
msg: e.to_string(),
span: Some(col.span),
help: None,
inner: vec![],
})
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(DropDF {})])
}
}

119
crates/nu-cmd-dataframe/src/dataframe/eager/drop_duplicates.rs Normal file
View File

@ -0,0 +1,119 @@
use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::UniqueKeepStrategy;
#[derive(Clone)]
pub struct DropDuplicates;
impl Command for DropDuplicates {
fn name(&self) -> &str {
"dfr drop-duplicates"
}
fn usage(&self) -> &str {
"Drops duplicate values in dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.optional(
"subset",
SyntaxShape::Table(vec![]),
"subset of columns to drop duplicates",
)
.switch("maintain", "maintain order", Some('m'))
.switch(
"last",
"keeps last duplicate value (by default keeps first)",
Some('l'),
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "drop duplicates",
example: "[[a b]; [1 2] [3 4] [1 2]] | dfr into-df | dfr drop-duplicates",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(3), Value::test_int(1)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(4), Value::test_int(2)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let columns: Option<Vec<Value>> = call.opt(engine_state, stack, 0)?;
let (subset, col_span) = match columns {
Some(cols) => {
let (agg_string, col_span) = convert_columns_string(cols, call.head)?;
(Some(agg_string), col_span)
}
None => (None, call.head),
};
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let subset_slice = subset.as_ref().map(|cols| &cols[..]);
let keep_strategy = if call.has_flag(engine_state, stack, "last")? {
UniqueKeepStrategy::Last
} else {
UniqueKeepStrategy::First
};
df.as_ref()
.unique(subset_slice, keep_strategy, None)
.map_err(|e| ShellError::GenericError {
error: "Error dropping duplicates".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(DropDuplicates {})])
}
}

137
crates/nu-cmd-dataframe/src/dataframe/eager/drop_nulls.rs Normal file
View File

@ -0,0 +1,137 @@
use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct DropNulls;
impl Command for DropNulls {
fn name(&self) -> &str {
"dfr drop-nulls"
}
fn usage(&self) -> &str {
"Drops null values in dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.optional(
"subset",
SyntaxShape::Table(vec![]),
"subset of columns to drop nulls",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "drop null values in dataframe",
example: r#"let df = ([[a b]; [1 2] [3 0] [1 2]] | dfr into-df);
let res = ($df.b / $df.b);
let a = ($df | dfr with-column $res --name res);
$a | dfr drop-nulls"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(1)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(2)],
),
Column::new(
"res".to_string(),
vec![Value::test_int(1), Value::test_int(1)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "drop null values in dataframe",
example: r#"let s = ([1 2 0 0 3 4] | dfr into-df);
($s / $s) | dfr drop-nulls"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"div_0_0".to_string(),
vec![
Value::test_int(1),
Value::test_int(1),
Value::test_int(1),
Value::test_int(1),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let columns: Option<Vec<Value>> = call.opt(engine_state, stack, 0)?;
let (subset, col_span) = match columns {
Some(cols) => {
let (agg_string, col_span) = convert_columns_string(cols, call.head)?;
(Some(agg_string), col_span)
}
None => (None, call.head),
};
let subset_slice = subset.as_ref().map(|cols| &cols[..]);
df.as_ref()
.drop_nulls(subset_slice)
.map_err(|e| ShellError::GenericError {
error: "Error dropping nulls".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::super::WithColumn;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(DropNulls {}), Box::new(WithColumn {})])
}
}

104
crates/nu-cmd-dataframe/src/dataframe/eager/dtypes.rs Normal file
View File

@ -0,0 +1,104 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct DataTypes;
impl Command for DataTypes {
fn name(&self) -> &str {
"dfr dtypes"
}
fn usage(&self) -> &str {
"Show dataframe data types."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Dataframe dtypes",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dtypes",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"column".to_string(),
vec![Value::test_string("a"), Value::test_string("b")],
),
Column::new(
"dtype".to_string(),
vec![Value::test_string("i64"), Value::test_string("i64")],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let mut dtypes: Vec<Value> = Vec::new();
let names: Vec<Value> = df
.as_ref()
.get_column_names()
.iter()
.map(|v| {
let dtype = df
.as_ref()
.column(v)
.expect("using name from list of names from dataframe")
.dtype();
let dtype_str = dtype.to_string();
dtypes.push(Value::string(dtype_str, call.head));
Value::string(*v, call.head)
})
.collect();
let names_col = Column::new("column".to_string(), names);
let dtypes_col = Column::new("dtype".to_string(), dtypes);
NuDataFrame::try_from_columns(vec![names_col, dtypes_col], None)
.map(|df| PipelineData::Value(df.into_value(call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(DataTypes {})])
}
}

107
crates/nu-cmd-dataframe/src/dataframe/eager/dummies.rs Normal file
View File

@ -0,0 +1,107 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::{prelude::*, series::Series};
#[derive(Clone)]
pub struct Dummies;
impl Command for Dummies {
fn name(&self) -> &str {
"dfr dummies"
}
fn usage(&self) -> &str {
"Creates a new dataframe with dummy variables."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.switch("drop-first", "Drop first row", Some('d'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Create new dataframe with dummy variables from a dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr dummies",
result: Some(
NuDataFrame::try_from_series(
vec![
Series::new("a_1", &[1_u8, 0]),
Series::new("a_3", &[0_u8, 1]),
Series::new("b_2", &[1_u8, 0]),
Series::new("b_4", &[0_u8, 1]),
],
Span::test_data(),
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Create new dataframe with dummy variables from a series",
example: "[1 2 2 3 3] | dfr into-df | dfr dummies",
result: Some(
NuDataFrame::try_from_series(
vec![
Series::new("0_1", &[1_u8, 0, 0, 0, 0]),
Series::new("0_2", &[0_u8, 1, 1, 0, 0]),
Series::new("0_3", &[0_u8, 0, 0, 1, 1]),
],
Span::test_data(),
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let drop_first: bool = call.has_flag(engine_state, stack, "drop-first")?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
df.as_ref()
.to_dummies(None, drop_first)
.map_err(|e| ShellError::GenericError {
error: "Error calculating dummies".into(),
msg: e.to_string(),
span: Some(call.head),
help: Some("The only allowed column types for dummies are String or Int".into()),
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(Dummies {})])
}
}

155
crates/nu-cmd-dataframe/src/dataframe/eager/filter_with.rs Normal file
View File

@ -0,0 +1,155 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
use polars::prelude::LazyFrame;
#[derive(Clone)]
pub struct FilterWith;
impl Command for FilterWith {
fn name(&self) -> &str {
"dfr filter-with"
}
fn usage(&self) -> &str {
"Filters dataframe using a mask or expression as reference."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"mask or expression",
SyntaxShape::Any,
"boolean mask used to filter data",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe or lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Filter dataframe using a bool mask",
example: r#"let mask = ([true false] | dfr into-df);
[[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with $mask"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(1)]),
Column::new("b".to_string(), vec![Value::test_int(2)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Filter dataframe using an expression",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr filter-with ((dfr col a) > 1)",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(3)]),
Column::new("b".to_string(), vec![Value::test_int(4)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head);
if NuLazyFrame::can_downcast(&value) {
let df = NuLazyFrame::try_from_value(value)?;
command_lazy(engine_state, stack, call, df)
} else {
let df = NuDataFrame::try_from_value(value)?;
command_eager(engine_state, stack, call, df)
}
}
}
fn command_eager(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let mask_value: Value = call.req(engine_state, stack, 0)?;
let mask_span = mask_value.span();
if NuExpression::can_downcast(&mask_value) {
let expression = NuExpression::try_from_value(mask_value)?;
let lazy = NuLazyFrame::new(true, df.lazy());
let lazy = lazy.apply_with_expr(expression, LazyFrame::filter);
Ok(PipelineData::Value(
NuLazyFrame::into_value(lazy, call.head)?,
None,
))
} else {
let mask = NuDataFrame::try_from_value(mask_value)?.as_series(mask_span)?;
let mask = mask.bool().map_err(|e| ShellError::GenericError {
error: "Error casting to bool".into(),
msg: e.to_string(),
span: Some(mask_span),
help: Some("Perhaps you want to use a series with booleans as mask".into()),
inner: vec![],
})?;
df.as_ref()
.filter(mask)
.map_err(|e| ShellError::GenericError {
error: "Error filtering dataframe".into(),
msg: e.to_string(),
span: Some(call.head),
help: Some("The only allowed column types for dummies are String or Int".into()),
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
}
fn command_lazy(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
lazy: NuLazyFrame,
) -> Result<PipelineData, ShellError> {
let expr: Value = call.req(engine_state, stack, 0)?;
let expr = NuExpression::try_from_value(expr)?;
let lazy = lazy.apply_with_expr(expr, LazyFrame::filter);
Ok(PipelineData::Value(
NuLazyFrame::into_value(lazy, call.head)?,
None,
))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::expressions::ExprCol;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(FilterWith {}), Box::new(ExprCol {})])
}
}

144
crates/nu-cmd-dataframe/src/dataframe/eager/first.rs Normal file
View File

@ -0,0 +1,144 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct FirstDF;
impl Command for FirstDF {
fn name(&self) -> &str {
"dfr first"
}
fn usage(&self) -> &str {
"Show only the first number of rows or create a first expression"
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.optional(
"rows",
SyntaxShape::Int,
"starting from the front, the number of rows to return",
)
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Return the first row of a dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(1)]),
Column::new("b".to_string(), vec![Value::test_int(2)]),
],
None,
)
.expect("should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Return the first two rows of a dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr first 2",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Creates a first expression from a column",
example: "dfr col a | dfr first",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head);
if NuDataFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value(value)?;
command(engine_state, stack, call, df)
} else {
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().first().into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let rows: Option<usize> = call.opt(engine_state, stack, 0)?;
let rows = rows.unwrap_or(1);
let res = df.as_ref().head(Some(rows));
Ok(PipelineData::Value(
NuDataFrame::dataframe_into_value(res, call.head),
None,
))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples_dataframe() {
let mut engine_state = build_test_engine_state(vec![Box::new(FirstDF {})]);
test_dataframe_example(&mut engine_state, &FirstDF.examples()[0]);
test_dataframe_example(&mut engine_state, &FirstDF.examples()[1]);
}
#[test]
fn test_examples_expression() {
let mut engine_state = build_test_engine_state(vec![
Box::new(FirstDF {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
]);
test_dataframe_example(&mut engine_state, &FirstDF.examples()[2]);
}
}

87
crates/nu-cmd-dataframe/src/dataframe/eager/get.rs Normal file
View File

@ -0,0 +1,87 @@
use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct GetDF;
impl Command for GetDF {
fn name(&self) -> &str {
"dfr get"
}
fn usage(&self) -> &str {
"Creates dataframe with the selected columns."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.rest("rest", SyntaxShape::Any, "column names to sort dataframe")
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Returns the selected column",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr get a",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let columns: Vec<Value> = call.rest(engine_state, stack, 0)?;
let (col_string, col_span) = convert_columns_string(columns, call.head)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
df.as_ref()
.select(col_string)
.map_err(|e| ShellError::GenericError {
error: "Error selecting columns".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(GetDF {})])
}
}

118
crates/nu-cmd-dataframe/src/dataframe/eager/last.rs Normal file
View File

@ -0,0 +1,118 @@
use crate::dataframe::values::{utils::DEFAULT_ROWS, Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct LastDF;
impl Command for LastDF {
fn name(&self) -> &str {
"dfr last"
}
fn usage(&self) -> &str {
"Creates new dataframe with tail rows or creates a last expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.optional("rows", SyntaxShape::Int, "Number of rows for tail")
.input_output_types(vec![
(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
),
(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
),
])
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Create new dataframe with last rows",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr last 1",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(3)]),
Column::new("b".to_string(), vec![Value::test_int(4)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Creates a last expression from a column",
example: "dfr col a | dfr last",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head);
if NuDataFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value(value)?;
command(engine_state, stack, call, df)
} else {
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = expr.into_polars().last().into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let rows: Option<usize> = call.opt(engine_state, stack, 0)?;
let rows = rows.unwrap_or(DEFAULT_ROWS);
let res = df.as_ref().tail(Some(rows));
Ok(PipelineData::Value(
NuDataFrame::dataframe_into_value(res, call.head),
None,
))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::{build_test_engine_state, test_dataframe_example};
use super::*;
use crate::dataframe::lazy::aggregate::LazyAggregate;
use crate::dataframe::lazy::groupby::ToLazyGroupBy;
#[test]
fn test_examples_dataframe() {
let mut engine_state = build_test_engine_state(vec![Box::new(LastDF {})]);
test_dataframe_example(&mut engine_state, &LastDF.examples()[0]);
}
#[test]
fn test_examples_expression() {
let mut engine_state = build_test_engine_state(vec![
Box::new(LastDF {}),
Box::new(LazyAggregate {}),
Box::new(ToLazyGroupBy {}),
]);
test_dataframe_example(&mut engine_state, &LastDF.examples()[1]);
}
}

68
crates/nu-cmd-dataframe/src/dataframe/eager/list.rs Normal file
View File

@ -0,0 +1,68 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ListDF;
impl Command for ListDF {
fn name(&self) -> &str {
"dfr ls"
}
fn usage(&self) -> &str {
"Lists stored dataframes."
}
fn signature(&self) -> Signature {
Signature::build(self.name()).category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Creates a new dataframe and shows it in the dataframe list",
example: r#"let test = ([[a b];[1 2] [3 4]] | dfr into-df);
ls"#,
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let mut vals: Vec<(String, Value)> = vec![];
for overlay_frame in engine_state.active_overlays(&[]) {
for var in &overlay_frame.vars {
if let Ok(value) = stack.get_var(*var.1, call.head) {
let name = String::from_utf8_lossy(var.0).to_string();
vals.push((name, value));
}
}
}
let vals = vals
.into_iter()
.filter_map(|(name, value)| {
NuDataFrame::try_from_value(value).ok().map(|df| (name, df))
})
.map(|(name, df)| {
Value::record(
record! {
"name" => Value::string(name, call.head),
"columns" => Value::int(df.as_ref().width() as i64, call.head),
"rows" => Value::int(df.as_ref().height() as i64, call.head),
},
call.head,
)
})
.collect::<Vec<Value>>();
let list = Value::list(vals, call.head);
Ok(list.into_pipeline_data())
}
}

248
crates/nu-cmd-dataframe/src/dataframe/eager/melt.rs Normal file
View File

@ -0,0 +1,248 @@
use crate::dataframe::values::{utils::convert_columns_string, Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct MeltDF;
impl Command for MeltDF {
fn name(&self) -> &str {
"dfr melt"
}
fn usage(&self) -> &str {
"Unpivot a DataFrame from wide to long format."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required_named(
"columns",
SyntaxShape::Table(vec![]),
"column names for melting",
Some('c'),
)
.required_named(
"values",
SyntaxShape::Table(vec![]),
"column names used as value columns",
Some('v'),
)
.named(
"variable-name",
SyntaxShape::String,
"optional name for variable column",
Some('r'),
)
.named(
"value-name",
SyntaxShape::String,
"optional name for value column",
Some('l'),
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "melt dataframe",
example:
"[[a b c d]; [x 1 4 a] [y 2 5 b] [z 3 6 c]] | dfr into-df | dfr melt -c [b c] -v [a d]",
result: Some(
NuDataFrame::try_from_columns(vec![
Column::new(
"b".to_string(),
vec![
Value::test_int(1),
Value::test_int(2),
Value::test_int(3),
Value::test_int(1),
Value::test_int(2),
Value::test_int(3),
],
),
Column::new(
"c".to_string(),
vec![
Value::test_int(4),
Value::test_int(5),
Value::test_int(6),
Value::test_int(4),
Value::test_int(5),
Value::test_int(6),
],
),
Column::new(
"variable".to_string(),
vec![
Value::test_string("a"),
Value::test_string("a"),
Value::test_string("a"),
Value::test_string("d"),
Value::test_string("d"),
Value::test_string("d"),
],
),
Column::new(
"value".to_string(),
vec![
Value::test_string("x"),
Value::test_string("y"),
Value::test_string("z"),
Value::test_string("a"),
Value::test_string("b"),
Value::test_string("c"),
],
),
], None)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let id_col: Vec<Value> = call
.get_flag(engine_state, stack, "columns")?
.expect("required value");
let val_col: Vec<Value> = call
.get_flag(engine_state, stack, "values")?
.expect("required value");
let value_name: Option<Spanned<String>> = call.get_flag(engine_state, stack, "value-name")?;
let variable_name: Option<Spanned<String>> =
call.get_flag(engine_state, stack, "variable-name")?;
let (id_col_string, id_col_span) = convert_columns_string(id_col, call.head)?;
let (val_col_string, val_col_span) = convert_columns_string(val_col, call.head)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
check_column_datatypes(df.as_ref(), &id_col_string, id_col_span)?;
check_column_datatypes(df.as_ref(), &val_col_string, val_col_span)?;
let mut res = df
.as_ref()
.melt(&id_col_string, &val_col_string)
.map_err(|e| ShellError::GenericError {
error: "Error calculating melt".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
if let Some(name) = &variable_name {
res.rename("variable", &name.item)
.map_err(|e| ShellError::GenericError {
error: "Error renaming column".into(),
msg: e.to_string(),
span: Some(name.span),
help: None,
inner: vec![],
})?;
}
if let Some(name) = &value_name {
res.rename("value", &name.item)
.map_err(|e| ShellError::GenericError {
error: "Error renaming column".into(),
msg: e.to_string(),
span: Some(name.span),
help: None,
inner: vec![],
})?;
}
Ok(PipelineData::Value(
NuDataFrame::dataframe_into_value(res, call.head),
None,
))
}
fn check_column_datatypes<T: AsRef<str>>(
df: &polars::prelude::DataFrame,
cols: &[T],
col_span: Span,
) -> Result<(), ShellError> {
if cols.is_empty() {
return Err(ShellError::GenericError {
error: "Merge error".into(),
msg: "empty column list".into(),
span: Some(col_span),
help: None,
inner: vec![],
});
}
// Checking if they are same type
if cols.len() > 1 {
for w in cols.windows(2) {
let l_series = df
.column(w[0].as_ref())
.map_err(|e| ShellError::GenericError {
error: "Error selecting columns".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})?;
let r_series = df
.column(w[1].as_ref())
.map_err(|e| ShellError::GenericError {
error: "Error selecting columns".into(),
msg: e.to_string(),
span: Some(col_span),
help: None,
inner: vec![],
})?;
if l_series.dtype() != r_series.dtype() {
return Err(ShellError::GenericError {
error: "Merge error".into(),
msg: "found different column types in list".into(),
span: Some(col_span),
help: Some(format!(
"datatypes {} and {} are incompatible",
l_series.dtype(),
r_series.dtype()
)),
inner: vec![],
});
}
}
}
Ok(())
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(MeltDF {})])
}
}

114
crates/nu-cmd-dataframe/src/dataframe/eager/mod.rs Normal file
View File

@ -0,0 +1,114 @@
mod append;
mod cast;
mod columns;
mod drop;
mod drop_duplicates;
mod drop_nulls;
mod dtypes;
mod dummies;
mod filter_with;
mod first;
mod get;
mod last;
mod list;
mod melt;
mod open;
mod query_df;
mod rename;
mod sample;
mod schema;
mod shape;
mod slice;
mod sql_context;
mod sql_expr;
mod summary;
mod take;
mod to_arrow;
mod to_avro;
mod to_csv;
mod to_df;
mod to_json_lines;
mod to_nu;
mod to_parquet;
mod with_column;
use nu_protocol::engine::StateWorkingSet;
pub use self::open::OpenDataFrame;
pub use append::AppendDF;
pub use cast::CastDF;
pub use columns::ColumnsDF;
pub use drop::DropDF;
pub use drop_duplicates::DropDuplicates;
pub use drop_nulls::DropNulls;
pub use dtypes::DataTypes;
pub use dummies::Dummies;
pub use filter_with::FilterWith;
pub use first::FirstDF;
pub use get::GetDF;
pub use last::LastDF;
pub use list::ListDF;
pub use melt::MeltDF;
pub use query_df::QueryDf;
pub use rename::RenameDF;
pub use sample::SampleDF;
pub use schema::SchemaDF;
pub use shape::ShapeDF;
pub use slice::SliceDF;
pub use sql_context::SQLContext;
pub use summary::Summary;
pub use take::TakeDF;
pub use to_arrow::ToArrow;
pub use to_avro::ToAvro;
pub use to_csv::ToCSV;
pub use to_df::ToDataFrame;
pub use to_json_lines::ToJsonLines;
pub use to_nu::ToNu;
pub use to_parquet::ToParquet;
pub use with_column::WithColumn;
pub fn add_eager_decls(working_set: &mut StateWorkingSet) {
macro_rules! bind_command {
( $command:expr ) => {
working_set.add_decl(Box::new($command));
};
( $( $command:expr ),* ) => {
$( working_set.add_decl(Box::new($command)); )*
};
}
// Dataframe commands
bind_command!(
AppendDF,
CastDF,
ColumnsDF,
DataTypes,
Summary,
DropDF,
DropDuplicates,
DropNulls,
Dummies,
FilterWith,
FirstDF,
GetDF,
LastDF,
ListDF,
MeltDF,
OpenDataFrame,
QueryDf,
RenameDF,
SampleDF,
SchemaDF,
ShapeDF,
SliceDF,
TakeDF,
ToArrow,
ToAvro,
ToCSV,
ToDataFrame,
ToNu,
ToParquet,
ToJsonLines,
WithColumn
);
}

518
crates/nu-cmd-dataframe/src/dataframe/eager/open.rs Normal file
View File

@ -0,0 +1,518 @@
use crate::dataframe::values::{NuDataFrame, NuLazyFrame, NuSchema};
use nu_engine::command_prelude::*;
use polars::prelude::{
CsvEncoding, CsvReader, IpcReader, JsonFormat, JsonReader, LazyCsvReader, LazyFileListReader,
LazyFrame, ParallelStrategy, ParquetReader, ScanArgsIpc, ScanArgsParquet, SerReader,
};
use polars_io::{avro::AvroReader, HiveOptions};
use std::{fs::File, io::BufReader, path::PathBuf};
#[derive(Clone)]
pub struct OpenDataFrame;
impl Command for OpenDataFrame {
fn name(&self) -> &str {
"dfr open"
}
fn usage(&self) -> &str {
"Opens CSV, JSON, JSON lines, arrow, avro, or parquet file to create dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"file",
SyntaxShape::Filepath,
"file path to load values from",
)
.switch("lazy", "creates a lazy dataframe", Some('l'))
.named(
"type",
SyntaxShape::String,
"File type: csv, tsv, json, parquet, arrow, avro. If omitted, derive from file extension",
Some('t'),
)
.named(
"delimiter",
SyntaxShape::String,
"file delimiter character. CSV file",
Some('d'),
)
.switch(
"no-header",
"Indicates if file doesn't have header. CSV file",
None,
)
.named(
"infer-schema",
SyntaxShape::Number,
"Number of rows to infer the schema of the file. CSV file",
None,
)
.named(
"skip-rows",
SyntaxShape::Number,
"Number of rows to skip from file. CSV file",
None,
)
.named(
"columns",
SyntaxShape::List(Box::new(SyntaxShape::String)),
"Columns to be selected from csv file. CSV and Parquet file",
None,
)
.named(
"schema",
SyntaxShape::Record(vec![]),
r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
Some('s')
)
.input_output_type(Type::Any, Type::Custom("dataframe".into()))
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Takes a file name and creates a dataframe",
example: "dfr open test.csv",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<PipelineData, ShellError> {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let type_option: Option<Spanned<String>> = call.get_flag(engine_state, stack, "type")?;
let type_id = match &type_option {
Some(ref t) => Some((t.item.to_owned(), "Invalid type", t.span)),
None => file.item.extension().map(|e| {
(
e.to_string_lossy().into_owned(),
"Invalid extension",
file.span,
)
}),
};
match type_id {
Some((e, msg, blamed)) => match e.as_str() {
"csv" | "tsv" => from_csv(engine_state, stack, call),
"parquet" | "parq" => from_parquet(engine_state, stack, call),
"ipc" | "arrow" => from_ipc(engine_state, stack, call),
"json" => from_json(engine_state, stack, call),
"jsonl" => from_jsonl(engine_state, stack, call),
"avro" => from_avro(engine_state, stack, call),
_ => Err(ShellError::FileNotFoundCustom {
msg: format!(
"{msg}. Supported values: csv, tsv, parquet, ipc, arrow, json, jsonl, avro"
),
span: blamed,
}),
},
None => Err(ShellError::FileNotFoundCustom {
msg: "File without extension".into(),
span: file.span,
}),
}
.map(|value| PipelineData::Value(value, None))
}
fn from_parquet(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
if call.has_flag(engine_state, stack, "lazy")? {
let file: String = call.req(engine_state, stack, 0)?;
let args = ScanArgsParquet {
n_rows: None,
cache: true,
parallel: ParallelStrategy::Auto,
rechunk: false,
row_index: None,
low_memory: false,
cloud_options: None,
use_statistics: false,
hive_options: HiveOptions::default(),
};
let df: NuLazyFrame = LazyFrame::scan_parquet(file, args)
.map_err(|e| ShellError::GenericError {
error: "Parquet reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
df.into_value(call.head)
} else {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let reader = ParquetReader::new(r);
let reader = match columns {
None => reader,
Some(columns) => reader.with_columns(Some(columns)),
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Parquet reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
}
fn from_avro(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let reader = AvroReader::new(r);
let reader = match columns {
None => reader,
Some(columns) => reader.with_columns(Some(columns)),
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Avro reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
fn from_ipc(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
if call.has_flag(engine_state, stack, "lazy")? {
let file: String = call.req(engine_state, stack, 0)?;
let args = ScanArgsIpc {
n_rows: None,
cache: true,
rechunk: false,
row_index: None,
memory_map: true,
cloud_options: None,
};
let df: NuLazyFrame = LazyFrame::scan_ipc(file, args)
.map_err(|e| ShellError::GenericError {
error: "IPC reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
df.into_value(call.head)
} else {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
let r = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let reader = IpcReader::new(r);
let reader = match columns {
None => reader,
Some(columns) => reader.with_columns(Some(columns)),
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "IPC reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
}
fn from_json(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let file = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let maybe_schema = call
.get_flag(engine_state, stack, "schema")?
.map(|schema| NuSchema::try_from(&schema))
.transpose()?;
let buf_reader = BufReader::new(file);
let reader = JsonReader::new(buf_reader);
let reader = match maybe_schema {
Some(schema) => reader.with_schema(schema.into()),
None => reader,
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Json reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
fn from_jsonl(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
let maybe_schema = call
.get_flag(engine_state, stack, "schema")?
.map(|schema| NuSchema::try_from(&schema))
.transpose()?;
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let file = File::open(&file.item).map_err(|e| ShellError::GenericError {
error: "Error opening file".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?;
let buf_reader = BufReader::new(file);
let reader = JsonReader::new(buf_reader)
.with_json_format(JsonFormat::JsonLines)
.infer_schema_len(infer_schema);
let reader = match maybe_schema {
Some(schema) => reader.with_schema(schema.into()),
None => reader,
};
let df: NuDataFrame = reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Json lines reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
fn from_csv(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
) -> Result<Value, ShellError> {
let delimiter: Option<Spanned<String>> = call.get_flag(engine_state, stack, "delimiter")?;
let no_header: bool = call.has_flag(engine_state, stack, "no-header")?;
let infer_schema: Option<usize> = call.get_flag(engine_state, stack, "infer-schema")?;
let skip_rows: Option<usize> = call.get_flag(engine_state, stack, "skip-rows")?;
let columns: Option<Vec<String>> = call.get_flag(engine_state, stack, "columns")?;
let maybe_schema = call
.get_flag(engine_state, stack, "schema")?
.map(|schema| NuSchema::try_from(&schema))
.transpose()?;
if call.has_flag(engine_state, stack, "lazy")? {
let file: String = call.req(engine_state, stack, 0)?;
let csv_reader = LazyCsvReader::new(file);
let csv_reader = match delimiter {
None => csv_reader,
Some(d) => {
if d.item.len() != 1 {
return Err(ShellError::GenericError {
error: "Incorrect delimiter".into(),
msg: "Delimiter has to be one character".into(),
span: Some(d.span),
help: None,
inner: vec![],
});
} else {
let delimiter = match d.item.chars().next() {
Some(d) => d as u8,
None => unreachable!(),
};
csv_reader.with_separator(delimiter)
}
}
};
let csv_reader = csv_reader.has_header(!no_header);
let csv_reader = match maybe_schema {
Some(schema) => csv_reader.with_schema(Some(schema.into())),
None => csv_reader,
};
let csv_reader = match infer_schema {
None => csv_reader,
Some(r) => csv_reader.with_infer_schema_length(Some(r)),
};
let csv_reader = match skip_rows {
None => csv_reader,
Some(r) => csv_reader.with_skip_rows(r),
};
let df: NuLazyFrame = csv_reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Parquet reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
df.into_value(call.head)
} else {
let file: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let csv_reader = CsvReader::from_path(&file.item)
.map_err(|e| ShellError::GenericError {
error: "Error creating CSV reader".into(),
msg: e.to_string(),
span: Some(file.span),
help: None,
inner: vec![],
})?
.with_encoding(CsvEncoding::LossyUtf8);
let csv_reader = match delimiter {
None => csv_reader,
Some(d) => {
if d.item.len() != 1 {
return Err(ShellError::GenericError {
error: "Incorrect delimiter".into(),
msg: "Delimiter has to be one character".into(),
span: Some(d.span),
help: None,
inner: vec![],
});
} else {
let delimiter = match d.item.chars().next() {
Some(d) => d as u8,
None => unreachable!(),
};
csv_reader.with_separator(delimiter)
}
}
};
let csv_reader = csv_reader.has_header(!no_header);
let csv_reader = match maybe_schema {
Some(schema) => csv_reader.with_schema(Some(schema.into())),
None => csv_reader,
};
let csv_reader = match infer_schema {
None => csv_reader,
Some(r) => csv_reader.infer_schema(Some(r)),
};
let csv_reader = match skip_rows {
None => csv_reader,
Some(r) => csv_reader.with_skip_rows(r),
};
let csv_reader = match columns {
None => csv_reader,
Some(columns) => csv_reader.with_columns(Some(columns)),
};
let df: NuDataFrame = csv_reader
.finish()
.map_err(|e| ShellError::GenericError {
error: "Parquet reader error".into(),
msg: format!("{e:?}"),
span: Some(call.head),
help: None,
inner: vec![],
})?
.into();
Ok(df.into_value(call.head))
}
}

104
crates/nu-cmd-dataframe/src/dataframe/eager/query_df.rs Normal file
View File

@ -0,0 +1,104 @@
use crate::dataframe::{
eager::SQLContext,
values::{Column, NuDataFrame, NuLazyFrame},
};
use nu_engine::command_prelude::*;
// attribution:
// sql_context.rs, and sql_expr.rs were copied from polars-sql. thank you.
// maybe we should just use the crate at some point but it's not published yet.
// https://github.com/pola-rs/polars/tree/master/polars-sql
#[derive(Clone)]
pub struct QueryDf;
impl Command for QueryDf {
fn name(&self) -> &str {
"dfr query"
}
fn usage(&self) -> &str {
"Query dataframe using SQL. Note: The dataframe is always named 'df' in your query's from clause."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("sql", SyntaxShape::String, "sql query")
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn search_terms(&self) -> Vec<&str> {
vec!["dataframe", "sql", "search"]
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Query dataframe using SQL",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr query 'select a from df'",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let sql_query: String = call.req(engine_state, stack, 0)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let mut ctx = SQLContext::new();
ctx.register("df", &df.df);
let df_sql = ctx
.execute(&sql_query)
.map_err(|e| ShellError::GenericError {
error: "Dataframe Error".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
let lazy = NuLazyFrame::new(false, df_sql);
let eager = lazy.collect(call.head)?;
let value = Value::custom(Box::new(eager), call.head);
Ok(PipelineData::Value(value, None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(QueryDf {})])
}
}

186
crates/nu-cmd-dataframe/src/dataframe/eager/rename.rs Normal file
View File

@ -0,0 +1,186 @@
use crate::dataframe::{
utils::extract_strings,
values::{Column, NuDataFrame, NuLazyFrame},
};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct RenameDF;
impl Command for RenameDF {
fn name(&self) -> &str {
"dfr rename"
}
fn usage(&self) -> &str {
"Rename a dataframe column."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"columns",
SyntaxShape::Any,
"Column(s) to be renamed. A string or list of strings",
)
.required(
"new names",
SyntaxShape::Any,
"New names for the selected column(s). A string or list of strings",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe or lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Renames a series",
example: "[5 6 7 8] | dfr into-df | dfr rename '0' new_name",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"new_name".to_string(),
vec![
Value::test_int(5),
Value::test_int(6),
Value::test_int(7),
Value::test_int(8),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Renames a dataframe column",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename a a_new",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a_new".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Renames two dataframe columns",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr rename [a b] [a_new b_new]",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a_new".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b_new".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head);
if NuLazyFrame::can_downcast(&value) {
let df = NuLazyFrame::try_from_value(value)?;
command_lazy(engine_state, stack, call, df)
} else {
let df = NuDataFrame::try_from_value(value)?;
command_eager(engine_state, stack, call, df)
}
}
}
fn command_eager(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
mut df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let columns: Value = call.req(engine_state, stack, 0)?;
let columns = extract_strings(columns)?;
let new_names: Value = call.req(engine_state, stack, 1)?;
let new_names = extract_strings(new_names)?;
for (from, to) in columns.iter().zip(new_names.iter()) {
df.as_mut()
.rename(from, to)
.map_err(|e| ShellError::GenericError {
error: "Error renaming".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})?;
}
Ok(PipelineData::Value(df.into_value(call.head), None))
}
fn command_lazy(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
lazy: NuLazyFrame,
) -> Result<PipelineData, ShellError> {
let columns: Value = call.req(engine_state, stack, 0)?;
let columns = extract_strings(columns)?;
let new_names: Value = call.req(engine_state, stack, 1)?;
let new_names = extract_strings(new_names)?;
if columns.len() != new_names.len() {
let value: Value = call.req(engine_state, stack, 1)?;
return Err(ShellError::IncompatibleParametersSingle {
msg: "New name list has different size to column list".into(),
span: value.span(),
});
}
let lazy = lazy.into_polars();
let lazy: NuLazyFrame = lazy.rename(&columns, &new_names).into();
Ok(PipelineData::Value(lazy.into_value(call.head)?, None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(RenameDF {})])
}
}

127
crates/nu-cmd-dataframe/src/dataframe/eager/sample.rs Normal file
View File

@ -0,0 +1,127 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::{prelude::NamedFrom, series::Series};
#[derive(Clone)]
pub struct SampleDF;
impl Command for SampleDF {
fn name(&self) -> &str {
"dfr sample"
}
fn usage(&self) -> &str {
"Create sample dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.named(
"n-rows",
SyntaxShape::Int,
"number of rows to be taken from dataframe",
Some('n'),
)
.named(
"fraction",
SyntaxShape::Number,
"fraction of dataframe to be taken",
Some('f'),
)
.named(
"seed",
SyntaxShape::Number,
"seed for the selection",
Some('s'),
)
.switch("replace", "sample with replace", Some('e'))
.switch("shuffle", "shuffle sample", Some('u'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Sample rows from dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr sample --n-rows 1",
result: None, // No expected value because sampling is random
},
Example {
description: "Shows sample row using fraction and replace",
example:
"[[a b]; [1 2] [3 4] [5 6]] | dfr into-df | dfr sample --fraction 0.5 --replace",
result: None, // No expected value because sampling is random
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let rows: Option<Spanned<i64>> = call.get_flag(engine_state, stack, "n-rows")?;
let fraction: Option<Spanned<f64>> = call.get_flag(engine_state, stack, "fraction")?;
let seed: Option<u64> = call
.get_flag::<i64>(engine_state, stack, "seed")?
.map(|val| val as u64);
let replace: bool = call.has_flag(engine_state, stack, "replace")?;
let shuffle: bool = call.has_flag(engine_state, stack, "shuffle")?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
match (rows, fraction) {
(Some(rows), None) => df
.as_ref()
.sample_n(&Series::new("s", &[rows.item]), replace, shuffle, seed)
.map_err(|e| ShellError::GenericError {
error: "Error creating sample".into(),
msg: e.to_string(),
span: Some(rows.span),
help: None,
inner: vec![],
}),
(None, Some(frac)) => df
.as_ref()
.sample_frac(&Series::new("frac", &[frac.item]), replace, shuffle, seed)
.map_err(|e| ShellError::GenericError {
error: "Error creating sample".into(),
msg: e.to_string(),
span: Some(frac.span),
help: None,
inner: vec![],
}),
(Some(_), Some(_)) => Err(ShellError::GenericError {
error: "Incompatible flags".into(),
msg: "Only one selection criterion allowed".into(),
span: Some(call.head),
help: None,
inner: vec![],
}),
(None, None) => Err(ShellError::GenericError {
error: "No selection".into(),
msg: "No selection criterion was found".into(),
span: Some(call.head),
help: Some("Perhaps you want to use the flag -n or -f".into()),
inner: vec![],
}),
}
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}

112
crates/nu-cmd-dataframe/src/dataframe/eager/schema.rs Normal file
View File

@ -0,0 +1,112 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct SchemaDF;
impl Command for SchemaDF {
fn name(&self) -> &str {
"dfr schema"
}
fn usage(&self) -> &str {
"Show schema for a dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.switch("datatype-list", "creates a lazy dataframe", Some('l'))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Dataframe schema",
example: r#"[[a b]; [1 "foo"] [3 "bar"]] | dfr into-df | dfr schema"#,
result: Some(Value::record(
record! {
"a" => Value::string("i64", Span::test_data()),
"b" => Value::string("str", Span::test_data()),
},
Span::test_data(),
)),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
if call.has_flag(engine_state, stack, "datatype-list")? {
Ok(PipelineData::Value(datatype_list(Span::unknown()), None))
} else {
command(engine_state, stack, call, input)
}
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let schema = df.schema();
let value: Value = schema.into();
Ok(PipelineData::Value(value, None))
}
fn datatype_list(span: Span) -> Value {
let types: Vec<Value> = [
("null", ""),
("bool", ""),
("u8", ""),
("u16", ""),
("u32", ""),
("u64", ""),
("i8", ""),
("i16", ""),
("i32", ""),
("i64", ""),
("f32", ""),
("f64", ""),
("str", ""),
("binary", ""),
("date", ""),
("datetime<time_unit: (ms, us, ns) timezone (optional)>", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns. Timezone wildcard is *. Other Timezone examples: UTC, America/Los_Angeles."),
("duration<time_unit: (ms, us, ns)>", "Time Unit can be: milliseconds: ms, microseconds: us, nanoseconds: ns."),
("time", ""),
("object", ""),
("unknown", ""),
("list<dtype>", ""),
]
.iter()
.map(|(dtype, note)| {
Value::record(record! {
"dtype" => Value::string(*dtype, span),
"note" => Value::string(*note, span),
},
span)
})
.collect();
Value::list(types, span)
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(SchemaDF {})])
}
}

82
crates/nu-cmd-dataframe/src/dataframe/eager/shape.rs Normal file
View File

@ -0,0 +1,82 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ShapeDF;
impl Command for ShapeDF {
fn name(&self) -> &str {
"dfr shape"
}
fn usage(&self) -> &str {
"Shows column and row size for a dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Shows row and column shape",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr shape",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("rows".to_string(), vec![Value::test_int(2)]),
Column::new("columns".to_string(), vec![Value::test_int(2)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
_engine_state: &EngineState,
_stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let rows = Value::int(df.as_ref().height() as i64, call.head);
let cols = Value::int(df.as_ref().width() as i64, call.head);
let rows_col = Column::new("rows".to_string(), vec![rows]);
let cols_col = Column::new("columns".to_string(), vec![cols]);
NuDataFrame::try_from_columns(vec![rows_col, cols_col], None)
.map(|df| PipelineData::Value(df.into_value(call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ShapeDF {})])
}
}

84
crates/nu-cmd-dataframe/src/dataframe/eager/slice.rs Normal file
View File

@ -0,0 +1,84 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct SliceDF;
impl Command for SliceDF {
fn name(&self) -> &str {
"dfr slice"
}
fn usage(&self) -> &str {
"Creates new dataframe from a slice of rows."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("offset", SyntaxShape::Int, "start of slice")
.required("size", SyntaxShape::Int, "size of slice")
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Create new dataframe from a slice of the rows",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr slice 0 1",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("a".to_string(), vec![Value::test_int(1)]),
Column::new("b".to_string(), vec![Value::test_int(2)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let offset: i64 = call.req(engine_state, stack, 0)?;
let size: usize = call.req(engine_state, stack, 1)?;
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let res = df.as_ref().slice(offset, size);
Ok(PipelineData::Value(
NuDataFrame::dataframe_into_value(res, call.head),
None,
))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(SliceDF {})])
}
}

228
crates/nu-cmd-dataframe/src/dataframe/eager/sql_context.rs Normal file
View File

@ -0,0 +1,228 @@
use crate::dataframe::eager::sql_expr::parse_sql_expr;
use polars::error::{ErrString, PolarsError};
use polars::prelude::{col, DataFrame, DataType, IntoLazy, LazyFrame};
use sqlparser::ast::{
Expr as SqlExpr, GroupByExpr, Select, SelectItem, SetExpr, Statement, TableFactor,
Value as SQLValue,
};
use sqlparser::dialect::GenericDialect;
use sqlparser::parser::Parser;
use std::collections::HashMap;
#[derive(Default)]
pub struct SQLContext {
table_map: HashMap<String, LazyFrame>,
dialect: GenericDialect,
}
impl SQLContext {
pub fn new() -> Self {
Self {
table_map: HashMap::new(),
dialect: GenericDialect,
}
}
pub fn register(&mut self, name: &str, df: &DataFrame) {
self.table_map.insert(name.to_owned(), df.clone().lazy());
}
fn execute_select(&self, select_stmt: &Select) -> Result<LazyFrame, PolarsError> {
// Determine involved dataframe
// Implicit join require some more work in query parsers, Explicit join are preferred for now.
let tbl = select_stmt.from.first().ok_or_else(|| {
PolarsError::ComputeError(ErrString::from("No table found in select statement"))
})?;
let mut alias_map = HashMap::new();
let tbl_name = match &tbl.relation {
TableFactor::Table { name, alias, .. } => {
let tbl_name = name
.0
.first()
.ok_or_else(|| {
PolarsError::ComputeError(ErrString::from(
"No table found in select statement",
))
})?
.value
.to_string();
if self.table_map.contains_key(&tbl_name) {
if let Some(alias) = alias {
alias_map.insert(alias.name.value.clone(), tbl_name.to_owned());
};
tbl_name
} else {
return Err(PolarsError::ComputeError(
format!("Table name {tbl_name} was not found").into(),
));
}
}
// Support bare table, optional with alias for now
_ => return Err(PolarsError::ComputeError("Not implemented".into())),
};
let df = &self.table_map[&tbl_name];
let mut raw_projection_before_alias: HashMap<String, usize> = HashMap::new();
let mut contain_wildcard = false;
// Filter Expression
let df = match select_stmt.selection.as_ref() {
Some(expr) => {
let filter_expression = parse_sql_expr(expr)?;
df.clone().filter(filter_expression)
}
None => df.clone(),
};
// Column Projections
let projection = select_stmt
.projection
.iter()
.enumerate()
.map(|(i, select_item)| {
Ok(match select_item {
SelectItem::UnnamedExpr(expr) => {
let expr = parse_sql_expr(expr)?;
raw_projection_before_alias.insert(format!("{expr:?}"), i);
expr
}
SelectItem::ExprWithAlias { expr, alias } => {
let expr = parse_sql_expr(expr)?;
raw_projection_before_alias.insert(format!("{expr:?}"), i);
expr.alias(&alias.value)
}
SelectItem::QualifiedWildcard(_, _) | SelectItem::Wildcard(_) => {
contain_wildcard = true;
col("*")
}
})
})
.collect::<Result<Vec<_>, PolarsError>>()?;
// Check for group by
// After projection since there might be number.
let group_by = match &select_stmt.group_by {
GroupByExpr::All =>
Err(
PolarsError::ComputeError("Group-By Error: Only positive number or expression are supported, not all".into())
)?,
GroupByExpr::Expressions(expressions) => expressions
}
.iter()
.map(
|e|match e {
SqlExpr::Value(SQLValue::Number(idx, _)) => {
let idx = match idx.parse::<usize>() {
Ok(0)| Err(_) => Err(
PolarsError::ComputeError(
format!("Group-By Error: Only positive number or expression are supported, got {idx}").into()
)),
Ok(idx) => Ok(idx)
}?;
Ok(projection[idx].clone())
}
SqlExpr::Value(_) => Err(
PolarsError::ComputeError("Group-By Error: Only positive number or expression are supported".into())
),
_ => parse_sql_expr(e)
}
)
.collect::<Result<Vec<_>, PolarsError>>()?;
let df = if group_by.is_empty() {
df.select(projection)
} else {
// check groupby and projection due to difference between SQL and polars
// Return error on wild card, shouldn't process this
if contain_wildcard {
return Err(PolarsError::ComputeError(
"Group-By Error: Can't process wildcard in group-by".into(),
));
}
// Default polars group by will have group by columns at the front
// need some container to contain position of group by columns and its position
// at the final agg projection, check the schema for the existence of group by column
// and its projections columns, keeping the original index
let (exclude_expr, groupby_pos): (Vec<_>, Vec<_>) = group_by
.iter()
.map(|expr| raw_projection_before_alias.get(&format!("{expr:?}")))
.enumerate()
.filter(|(_, proj_p)| proj_p.is_some())
.map(|(gb_p, proj_p)| (*proj_p.unwrap_or(&0), (*proj_p.unwrap_or(&0), gb_p)))
.unzip();
let (agg_projection, agg_proj_pos): (Vec<_>, Vec<_>) = projection
.iter()
.enumerate()
.filter(|(i, _)| !exclude_expr.contains(i))
.enumerate()
.map(|(agg_pj, (proj_p, expr))| (expr.clone(), (proj_p, agg_pj + group_by.len())))
.unzip();
let agg_df = df.group_by(group_by).agg(agg_projection);
let mut final_proj_pos = groupby_pos
.into_iter()
.chain(agg_proj_pos)
.collect::<Vec<_>>();
final_proj_pos.sort_by(|(proj_pa, _), (proj_pb, _)| proj_pa.cmp(proj_pb));
let final_proj = final_proj_pos
.into_iter()
.map(|(_, shm_p)| {
col(agg_df
.clone()
// FIXME: had to do this mess to get get_index to work, not sure why. need help
.collect()
.unwrap_or_default()
.schema()
.get_at_index(shm_p)
.unwrap_or((&"".into(), &DataType::Null))
.0)
})
.collect::<Vec<_>>();
agg_df.select(final_proj)
};
Ok(df)
}
pub fn execute(&self, query: &str) -> Result<LazyFrame, PolarsError> {
let ast = Parser::parse_sql(&self.dialect, query)
.map_err(|e| PolarsError::ComputeError(format!("{e:?}").into()))?;
if ast.len() != 1 {
Err(PolarsError::ComputeError(
"One and only one statement at a time please".into(),
))
} else {
let ast = ast
.first()
.ok_or_else(|| PolarsError::ComputeError(ErrString::from("No statement found")))?;
Ok(match ast {
Statement::Query(query) => {
let rs = match &*query.body {
SetExpr::Select(select_stmt) => self.execute_select(select_stmt)?,
_ => {
return Err(PolarsError::ComputeError(
"INSERT, UPDATE is not supported for polars".into(),
))
}
};
match &query.limit {
Some(SqlExpr::Value(SQLValue::Number(nrow, _))) => {
let nrow = nrow.parse().map_err(|err| {
PolarsError::ComputeError(
format!("Conversion Error: {err:?}").into(),
)
})?;
rs.limit(nrow)
}
None => rs,
_ => {
return Err(PolarsError::ComputeError(
"Only support number argument to LIMIT clause".into(),
))
}
}
}
_ => {
return Err(PolarsError::ComputeError(
format!("Statement type {ast:?} is not supported").into(),
))
}
})
}
}
}

200
crates/nu-cmd-dataframe/src/dataframe/eager/sql_expr.rs Normal file
View File

@ -0,0 +1,200 @@
use polars::error::PolarsError;
use polars::prelude::{col, lit, DataType, Expr, LiteralValue, PolarsResult as Result, TimeUnit};
use sqlparser::ast::{
ArrayElemTypeDef, BinaryOperator as SQLBinaryOperator, DataType as SQLDataType,
Expr as SqlExpr, Function as SQLFunction, Value as SqlValue, WindowType,
};
fn map_sql_polars_datatype(data_type: &SQLDataType) -> Result<DataType> {
Ok(match data_type {
SQLDataType::Char(_)
| SQLDataType::Varchar(_)
| SQLDataType::Uuid
| SQLDataType::Clob(_)
| SQLDataType::Text
| SQLDataType::String(_) => DataType::String,
SQLDataType::Float(_) => DataType::Float32,
SQLDataType::Real => DataType::Float32,
SQLDataType::Double => DataType::Float64,
SQLDataType::TinyInt(_) => DataType::Int8,
SQLDataType::UnsignedTinyInt(_) => DataType::UInt8,
SQLDataType::SmallInt(_) => DataType::Int16,
SQLDataType::UnsignedSmallInt(_) => DataType::UInt16,
SQLDataType::Int(_) => DataType::Int32,
SQLDataType::UnsignedInt(_) => DataType::UInt32,
SQLDataType::BigInt(_) => DataType::Int64,
SQLDataType::UnsignedBigInt(_) => DataType::UInt64,
SQLDataType::Boolean => DataType::Boolean,
SQLDataType::Date => DataType::Date,
SQLDataType::Time(_, _) => DataType::Time,
SQLDataType::Timestamp(_, _) => DataType::Datetime(TimeUnit::Microseconds, None),
SQLDataType::Interval => DataType::Duration(TimeUnit::Microseconds),
SQLDataType::Array(array_type_def) => match array_type_def {
ArrayElemTypeDef::AngleBracket(inner_type)
| ArrayElemTypeDef::SquareBracket(inner_type) => {
DataType::List(Box::new(map_sql_polars_datatype(inner_type)?))
}
_ => {
return Err(PolarsError::ComputeError(
"SQL Datatype Array(None) was not supported in polars-sql yet!".into(),
))
}
},
_ => {
return Err(PolarsError::ComputeError(
format!("SQL Datatype {data_type:?} was not supported in polars-sql yet!").into(),
))
}
})
}
fn cast_(expr: Expr, data_type: &SQLDataType) -> Result<Expr> {
let polars_type = map_sql_polars_datatype(data_type)?;
Ok(expr.cast(polars_type))
}
fn binary_op_(left: Expr, right: Expr, op: &SQLBinaryOperator) -> Result<Expr> {
Ok(match op {
SQLBinaryOperator::Plus => left + right,
SQLBinaryOperator::Minus => left - right,
SQLBinaryOperator::Multiply => left * right,
SQLBinaryOperator::Divide => left / right,
SQLBinaryOperator::Modulo => left % right,
SQLBinaryOperator::StringConcat => {
left.cast(DataType::String) + right.cast(DataType::String)
}
SQLBinaryOperator::Gt => left.gt(right),
SQLBinaryOperator::Lt => left.lt(right),
SQLBinaryOperator::GtEq => left.gt_eq(right),
SQLBinaryOperator::LtEq => left.lt_eq(right),
SQLBinaryOperator::Eq => left.eq(right),
SQLBinaryOperator::NotEq => left.eq(right).not(),
SQLBinaryOperator::And => left.and(right),
SQLBinaryOperator::Or => left.or(right),
SQLBinaryOperator::Xor => left.xor(right),
_ => {
return Err(PolarsError::ComputeError(
format!("SQL Operator {op:?} was not supported in polars-sql yet!").into(),
))
}
})
}
fn literal_expr(value: &SqlValue) -> Result<Expr> {
Ok(match value {
SqlValue::Number(s, _) => {
// Check for existence of decimal separator dot
if s.contains('.') {
s.parse::<f64>().map(lit).map_err(|_| {
PolarsError::ComputeError(format!("Can't parse literal {s:?}").into())
})
} else {
s.parse::<i64>().map(lit).map_err(|_| {
PolarsError::ComputeError(format!("Can't parse literal {s:?}").into())
})
}?
}
SqlValue::SingleQuotedString(s) => lit(s.clone()),
SqlValue::NationalStringLiteral(s) => lit(s.clone()),
SqlValue::HexStringLiteral(s) => lit(s.clone()),
SqlValue::DoubleQuotedString(s) => lit(s.clone()),
SqlValue::Boolean(b) => lit(*b),
SqlValue::Null => Expr::Literal(LiteralValue::Null),
_ => {
return Err(PolarsError::ComputeError(
format!("Parsing SQL Value {value:?} was not supported in polars-sql yet!").into(),
))
}
})
}
pub fn parse_sql_expr(expr: &SqlExpr) -> Result<Expr> {
Ok(match expr {
SqlExpr::Identifier(e) => col(&e.value),
SqlExpr::BinaryOp { left, op, right } => {
let left = parse_sql_expr(left)?;
let right = parse_sql_expr(right)?;
binary_op_(left, right, op)?
}
SqlExpr::Function(sql_function) => parse_sql_function(sql_function)?,
SqlExpr::Cast {
expr,
data_type,
format: _,
} => cast_(parse_sql_expr(expr)?, data_type)?,
SqlExpr::Nested(expr) => parse_sql_expr(expr)?,
SqlExpr::Value(value) => literal_expr(value)?,
_ => {
return Err(PolarsError::ComputeError(
format!("Expression: {expr:?} was not supported in polars-sql yet!").into(),
))
}
})
}
fn apply_window_spec(expr: Expr, window_type: Option<&WindowType>) -> Result<Expr> {
Ok(match &window_type {
Some(wtype) => match wtype {
WindowType::WindowSpec(window_spec) => {
// Process for simple window specification, partition by first
let partition_by = window_spec
.partition_by
.iter()
.map(parse_sql_expr)
.collect::<Result<Vec<_>>>()?;
expr.over(partition_by)
// Order by and Row range may not be supported at the moment
}
// TODO: make NamedWindow work
WindowType::NamedWindow(_named) => {
return Err(PolarsError::ComputeError(
format!("Expression: {expr:?} was not supported in polars-sql yet!").into(),
))
}
},
None => expr,
})
}
fn parse_sql_function(sql_function: &SQLFunction) -> Result<Expr> {
use sqlparser::ast::{FunctionArg, FunctionArgExpr};
// Function name mostly do not have name space, so it mostly take the first args
let function_name = sql_function.name.0[0].value.to_ascii_lowercase();
let args = sql_function
.args
.iter()
.map(|arg| match arg {
FunctionArg::Named { arg, .. } => arg,
FunctionArg::Unnamed(arg) => arg,
})
.collect::<Vec<_>>();
Ok(
match (
function_name.as_str(),
args.as_slice(),
sql_function.distinct,
) {
("sum", [FunctionArgExpr::Expr(expr)], false) => {
apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.sum()
}
("count", [FunctionArgExpr::Expr(expr)], false) => {
apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.count()
}
("count", [FunctionArgExpr::Expr(expr)], true) => {
apply_window_spec(parse_sql_expr(expr)?, sql_function.over.as_ref())?.n_unique()
}
// Special case for wildcard args to count function.
("count", [FunctionArgExpr::Wildcard], false) => lit(1i32).count(),
_ => {
return Err(PolarsError::ComputeError(
format!(
"Function {function_name:?} with args {args:?} was not supported in polars-sql yet!"
)
.into(),
))
}
},
)
}

279
crates/nu-cmd-dataframe/src/dataframe/eager/summary.rs Normal file
View File

@ -0,0 +1,279 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::{
chunked_array::ChunkedArray,
prelude::{
AnyValue, DataFrame, DataType, Float64Type, IntoSeries, NewChunkedArray,
QuantileInterpolOptions, Series, StringType,
},
};
#[derive(Clone)]
pub struct Summary;
impl Command for Summary {
fn name(&self) -> &str {
"dfr summary"
}
fn usage(&self) -> &str {
"For a dataframe, produces descriptive statistics (summary statistics) for its numeric columns."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.category(Category::Custom("dataframe".into()))
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.named(
"quantiles",
SyntaxShape::Table(vec![]),
"provide optional quantiles",
Some('q'),
)
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "list dataframe descriptives",
example: "[[a b]; [1 1] [1 1]] | dfr into-df | dfr summary",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"descriptor".to_string(),
vec![
Value::test_string("count"),
Value::test_string("sum"),
Value::test_string("mean"),
Value::test_string("median"),
Value::test_string("std"),
Value::test_string("min"),
Value::test_string("25%"),
Value::test_string("50%"),
Value::test_string("75%"),
Value::test_string("max"),
],
),
Column::new(
"a (i64)".to_string(),
vec![
Value::test_float(2.0),
Value::test_float(2.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(0.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
],
),
Column::new(
"b (i64)".to_string(),
vec![
Value::test_float(2.0),
Value::test_float(2.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(0.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
Value::test_float(1.0),
],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let quantiles: Option<Vec<Value>> = call.get_flag(engine_state, stack, "quantiles")?;
let quantiles = quantiles.map(|values| {
values
.iter()
.map(|value| {
let span = value.span();
match value {
Value::Float { val, .. } => {
if (&0.0..=&1.0).contains(&val) {
Ok(*val)
} else {
Err(ShellError::GenericError {
error: "Incorrect value for quantile".into(),
msg: "value should be between 0 and 1".into(),
span: Some(span),
help: None,
inner: vec![],
})
}
}
Value::Error { error, .. } => Err(*error.clone()),
_ => Err(ShellError::GenericError {
error: "Incorrect value for quantile".into(),
msg: "value should be a float".into(),
span: Some(span),
help: None,
inner: vec![],
}),
}
})
.collect::<Result<Vec<f64>, ShellError>>()
});
let quantiles = match quantiles {
Some(quantiles) => quantiles?,
None => vec![0.25, 0.50, 0.75],
};
let mut quantiles_labels = quantiles
.iter()
.map(|q| Some(format!("{}%", q * 100.0)))
.collect::<Vec<Option<String>>>();
let mut labels = vec![
Some("count".to_string()),
Some("sum".to_string()),
Some("mean".to_string()),
Some("median".to_string()),
Some("std".to_string()),
Some("min".to_string()),
];
labels.append(&mut quantiles_labels);
labels.push(Some("max".to_string()));
let df = NuDataFrame::try_from_pipeline(input, call.head)?;
let names = ChunkedArray::<StringType>::from_slice_options("descriptor", &labels).into_series();
let head = std::iter::once(names);
let tail = df
.as_ref()
.get_columns()
.iter()
.filter(|col| !matches!(col.dtype(), &DataType::Object("object", _)))
.map(|col| {
let count = col.len() as f64;
let sum = col.sum_as_series().ok().and_then(|series| {
series
.cast(&DataType::Float64)
.ok()
.and_then(|ca| match ca.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
})
});
let mean = match col.mean_as_series().get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
};
let median = match col.median_as_series() {
Ok(v) => match v.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
},
_ => None,
};
let std = match col.std_as_series(0) {
Ok(v) => match v.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
},
_ => None,
};
let min = col.min_as_series().ok().and_then(|series| {
series
.cast(&DataType::Float64)
.ok()
.and_then(|ca| match ca.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
})
});
let mut quantiles = quantiles
.clone()
.into_iter()
.map(|q| {
col.quantile_as_series(q, QuantileInterpolOptions::default())
.ok()
.and_then(|ca| ca.cast(&DataType::Float64).ok())
.and_then(|ca| match ca.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
})
})
.collect::<Vec<Option<f64>>>();
let max = col.max_as_series().ok().and_then(|series| {
series
.cast(&DataType::Float64)
.ok()
.and_then(|ca| match ca.get(0) {
Ok(AnyValue::Float64(v)) => Some(v),
_ => None,
})
});
let mut descriptors = vec![Some(count), sum, mean, median, std, min];
descriptors.append(&mut quantiles);
descriptors.push(max);
let name = format!("{} ({})", col.name(), col.dtype());
ChunkedArray::<Float64Type>::from_slice_options(&name, &descriptors).into_series()
});
let res = head.chain(tail).collect::<Vec<Series>>();
DataFrame::new(res)
.map_err(|e| ShellError::GenericError {
error: "Dataframe Error".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(Summary {})])
}
}

148
crates/nu-cmd-dataframe/src/dataframe/eager/take.rs Normal file
View File

@ -0,0 +1,148 @@
use crate::dataframe::values::{Column, NuDataFrame};
use nu_engine::command_prelude::*;
use polars::prelude::DataType;
#[derive(Clone)]
pub struct TakeDF;
impl Command for TakeDF {
fn name(&self) -> &str {
"dfr take"
}
fn usage(&self) -> &str {
"Creates new dataframe using the given indices."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"indices",
SyntaxShape::Any,
"list of indices used to take data",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Takes selected rows from dataframe",
example: r#"let df = ([[a b]; [4 1] [5 2] [4 3]] | dfr into-df);
let indices = ([0 2] | dfr into-df);
$df | dfr take $indices"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(4), Value::test_int(4)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Takes selected rows from series",
example: r#"let series = ([4 1 5 2 4 3] | dfr into-df);
let indices = ([0 2] | dfr into-df);
$series | dfr take $indices"#,
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![Value::test_int(4), Value::test_int(5)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let index_value: Value = call.req(engine_state, stack, 0)?;
let index_span = index_value.span();
let index = NuDataFrame::try_from_value(index_value)?.as_series(index_span)?;
let casted = match index.dtype() {
DataType::UInt32 | DataType::UInt64 | DataType::Int32 | DataType::Int64 => index
.cast(&DataType::UInt32)
.map_err(|e| ShellError::GenericError {
error: "Error casting index list".into(),
msg: e.to_string(),
span: Some(index_span),
help: None,
inner: vec![],
}),
_ => Err(ShellError::GenericError {
error: "Incorrect type".into(),
msg: "Series with incorrect type".into(),
span: Some(call.head),
help: Some("Consider using a Series with type int type".into()),
inner: vec![],
}),
}?;
let indices = casted.u32().map_err(|e| ShellError::GenericError {
error: "Error casting index list".into(),
msg: e.to_string(),
span: Some(index_span),
help: None,
inner: vec![],
})?;
NuDataFrame::try_from_pipeline(input, call.head).and_then(|df| {
df.as_ref()
.take(indices)
.map_err(|e| ShellError::GenericError {
error: "Error taking values".into(),
msg: e.to_string(),
span: Some(call.head),
help: None,
inner: vec![],
})
.map(|df| PipelineData::Value(NuDataFrame::dataframe_into_value(df, call.head), None))
})
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(TakeDF {})])
}
}

79
crates/nu-cmd-dataframe/src/dataframe/eager/to_arrow.rs Normal file
View File

@ -0,0 +1,79 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::prelude::{IpcWriter, SerWriter};
use std::{fs::File, path::PathBuf};
#[derive(Clone)]
pub struct ToArrow;
impl Command for ToArrow {
fn name(&self) -> &str {
"dfr to-arrow"
}
fn usage(&self) -> &str {
"Saves dataframe to arrow file."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("file", SyntaxShape::Filepath, "file path to save dataframe")
.input_output_type(Type::Custom("dataframe".into()), Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Saves dataframe to arrow file",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-arrow test.arrow",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
let mut file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
error: "Error with file name".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
IpcWriter::new(&mut file)
.finish(df.as_mut())
.map_err(|e| ShellError::GenericError {
error: "Error saving file".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
Ok(PipelineData::Value(
Value::list(vec![file_value], call.head),
None,
))
}

109
crates/nu-cmd-dataframe/src/dataframe/eager/to_avro.rs Normal file
View File

@ -0,0 +1,109 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars_io::{
avro::{AvroCompression, AvroWriter},
SerWriter,
};
use std::{fs::File, path::PathBuf};
#[derive(Clone)]
pub struct ToAvro;
impl Command for ToAvro {
fn name(&self) -> &str {
"dfr to-avro"
}
fn usage(&self) -> &str {
"Saves dataframe to avro file."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.named(
"compression",
SyntaxShape::String,
"use compression, supports deflate or snappy",
Some('c'),
)
.required("file", SyntaxShape::Filepath, "file path to save dataframe")
.input_output_type(Type::Custom("dataframe".into()), Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Saves dataframe to avro file",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-avro test.avro",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn get_compression(call: &Call) -> Result<Option<AvroCompression>, ShellError> {
if let Some((compression, span)) = call
.get_flag_expr("compression")
.and_then(|e| e.as_string().map(|s| (s, e.span)))
{
match compression.as_ref() {
"snappy" => Ok(Some(AvroCompression::Snappy)),
"deflate" => Ok(Some(AvroCompression::Deflate)),
_ => Err(ShellError::IncorrectValue {
msg: "compression must be one of deflate or snappy".to_string(),
val_span: span,
call_span: span,
}),
}
} else {
Ok(None)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let compression = get_compression(call)?;
let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
error: "Error with file name".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
AvroWriter::new(file)
.with_compression(compression)
.finish(df.as_mut())
.map_err(|e| ShellError::GenericError {
error: "Error saving file".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
Ok(PipelineData::Value(
Value::list(vec![file_value], call.head),
None,
))
}

125
crates/nu-cmd-dataframe/src/dataframe/eager/to_csv.rs Normal file
View File

@ -0,0 +1,125 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::prelude::{CsvWriter, SerWriter};
use std::{fs::File, path::PathBuf};
#[derive(Clone)]
pub struct ToCSV;
impl Command for ToCSV {
fn name(&self) -> &str {
"dfr to-csv"
}
fn usage(&self) -> &str {
"Saves dataframe to CSV file."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("file", SyntaxShape::Filepath, "file path to save dataframe")
.named(
"delimiter",
SyntaxShape::String,
"file delimiter character",
Some('d'),
)
.switch("no-header", "Indicates if file doesn't have header", None)
.input_output_type(Type::Custom("dataframe".into()), Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Saves dataframe to CSV file",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-csv test.csv",
result: None,
},
Example {
description: "Saves dataframe to CSV file using other delimiter",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-csv test.csv --delimiter '|'",
result: None,
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let delimiter: Option<Spanned<String>> = call.get_flag(engine_state, stack, "delimiter")?;
let no_header: bool = call.has_flag(engine_state, stack, "no-header")?;
let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
let mut file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
error: "Error with file name".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let writer = CsvWriter::new(&mut file);
let writer = if no_header {
writer.include_header(false)
} else {
writer.include_header(true)
};
let mut writer = match delimiter {
None => writer,
Some(d) => {
if d.item.len() != 1 {
return Err(ShellError::GenericError {
error: "Incorrect delimiter".into(),
msg: "Delimiter has to be one char".into(),
span: Some(d.span),
help: None,
inner: vec![],
});
} else {
let delimiter = match d.item.chars().next() {
Some(d) => d as u8,
None => unreachable!(),
};
writer.with_separator(delimiter)
}
}
};
writer
.finish(df.as_mut())
.map_err(|e| ShellError::GenericError {
error: "Error writing to file".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
Ok(PipelineData::Value(
Value::list(vec![file_value], call.head),
None,
))
}

189
crates/nu-cmd-dataframe/src/dataframe/eager/to_df.rs Normal file
View File

@ -0,0 +1,189 @@
use crate::dataframe::values::{Column, NuDataFrame, NuSchema};
use nu_engine::command_prelude::*;
use polars::prelude::*;
#[derive(Clone)]
pub struct ToDataFrame;
impl Command for ToDataFrame {
fn name(&self) -> &str {
"dfr into-df"
}
fn usage(&self) -> &str {
"Converts a list, table or record into a dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.named(
"schema",
SyntaxShape::Record(vec![]),
r#"Polars Schema in format [{name: str}]. CSV, JSON, and JSONL files"#,
Some('s'),
)
.input_output_type(Type::Any, Type::Custom("dataframe".into()))
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Takes a dictionary and creates a dataframe",
example: "[[a b];[1 2] [3 4]] | dfr into-df",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Takes a list of tables and creates a dataframe",
example: "[[1 2 a] [3 4 b] [5 6 c]] | dfr into-df",
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"0".to_string(),
vec![Value::test_int(1), Value::test_int(3), Value::test_int(5)],
),
Column::new(
"1".to_string(),
vec![Value::test_int(2), Value::test_int(4), Value::test_int(6)],
),
Column::new(
"2".to_string(),
vec![
Value::test_string("a"),
Value::test_string("b"),
Value::test_string("c"),
],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Takes a list and creates a dataframe",
example: "[a b c] | dfr into-df",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![
Value::test_string("a"),
Value::test_string("b"),
Value::test_string("c"),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Takes a list of booleans and creates a dataframe",
example: "[true true false] | dfr into-df",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"0".to_string(),
vec![
Value::test_bool(true),
Value::test_bool(true),
Value::test_bool(false),
],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Convert to a dataframe and provide a schema",
example: "{a: 1, b: {a: [1 2 3]}, c: [a b c]}| dfr into-df -s {a: u8, b: {a: list<u64>}, c: list<str>}",
result: Some(
NuDataFrame::try_from_series(vec![
Series::new("a", &[1u8]),
{
let dtype = DataType::Struct(vec![Field::new("a", DataType::List(Box::new(DataType::UInt64)))]);
let vals = vec![AnyValue::StructOwned(
Box::new((vec![AnyValue::List(Series::new("a", &[1u64, 2, 3]))], vec![Field::new("a", DataType::String)]))); 1];
Series::from_any_values_and_dtype("b", &vals, &dtype, false)
.expect("Struct series should not fail")
},
{
let dtype = DataType::List(Box::new(DataType::String));
let vals = vec![AnyValue::List(Series::new("c", &["a", "b", "c"]))];
Series::from_any_values_and_dtype("c", &vals, &dtype, false)
.expect("List series should not fail")
}
], Span::test_data())
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Convert to a dataframe and provide a schema that adds a new column",
example: r#"[[a b]; [1 "foo"] [2 "bar"]] | dfr into-df -s {a: u8, b:str, c:i64} | dfr fill-null 3"#,
result: Some(NuDataFrame::try_from_series(vec![
Series::new("a", [1u8, 2]),
Series::new("b", ["foo", "bar"]),
Series::new("c", [3i64, 3]),
], Span::test_data())
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let maybe_schema = call
.get_flag(engine_state, stack, "schema")?
.map(|schema| NuSchema::try_from(&schema))
.transpose()?;
let df = NuDataFrame::try_from_iter(input.into_iter(), maybe_schema.clone())?;
Ok(PipelineData::Value(
NuDataFrame::into_value(df, call.head),
None,
))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ToDataFrame {})])
}
}

80
crates/nu-cmd-dataframe/src/dataframe/eager/to_json_lines.rs Normal file
View File

@ -0,0 +1,80 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::prelude::{JsonWriter, SerWriter};
use std::{fs::File, io::BufWriter, path::PathBuf};
#[derive(Clone)]
pub struct ToJsonLines;
impl Command for ToJsonLines {
fn name(&self) -> &str {
"dfr to-jsonl"
}
fn usage(&self) -> &str {
"Saves dataframe to a JSON lines file."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("file", SyntaxShape::Filepath, "file path to save dataframe")
.input_output_type(Type::Custom("dataframe".into()), Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Saves dataframe to JSON lines file",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-jsonl test.jsonl",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
error: "Error with file name".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let buf_writer = BufWriter::new(file);
JsonWriter::new(buf_writer)
.finish(df.as_mut())
.map_err(|e| ShellError::GenericError {
error: "Error saving file".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
Ok(PipelineData::Value(
Value::list(vec![file_value], call.head),
None,
))
}

136
crates/nu-cmd-dataframe/src/dataframe/eager/to_nu.rs Normal file
View File

@ -0,0 +1,136 @@
use crate::dataframe::values::{NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ToNu;
impl Command for ToNu {
fn name(&self) -> &str {
"dfr into-nu"
}
fn usage(&self) -> &str {
"Converts a dataframe or an expression into into nushell value for access and exploration."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.named(
"rows",
SyntaxShape::Number,
"number of rows to be shown",
Some('n'),
)
.switch("tail", "shows tail rows", Some('t'))
.input_output_types(vec![
(Type::Custom("expression".into()), Type::Any),
(Type::Custom("dataframe".into()), Type::table()),
])
//.input_output_type(Type::Any, Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
let rec_1 = Value::test_record(record! {
"index" => Value::test_int(0),
"a" => Value::test_int(1),
"b" => Value::test_int(2),
});
let rec_2 = Value::test_record(record! {
"index" => Value::test_int(1),
"a" => Value::test_int(3),
"b" => Value::test_int(4),
});
let rec_3 = Value::test_record(record! {
"index" => Value::test_int(2),
"a" => Value::test_int(3),
"b" => Value::test_int(4),
});
vec![
Example {
description: "Shows head rows from dataframe",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr into-nu",
result: Some(Value::list(vec![rec_1, rec_2], Span::test_data())),
},
Example {
description: "Shows tail rows from dataframe",
example: "[[a b]; [1 2] [5 6] [3 4]] | dfr into-df | dfr into-nu --tail --rows 1",
result: Some(Value::list(vec![rec_3], Span::test_data())),
},
Example {
description: "Convert a col expression into a nushell value",
example: "dfr col a | dfr into-nu",
result: Some(Value::test_record(record! {
"expr" => Value::test_string("column"),
"value" => Value::test_string("a"),
})),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head);
if NuDataFrame::can_downcast(&value) {
dataframe_command(engine_state, stack, call, value)
} else {
expression_command(call, value)
}
}
}
fn dataframe_command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: Value,
) -> Result<PipelineData, ShellError> {
let rows: Option<usize> = call.get_flag(engine_state, stack, "rows")?;
let tail: bool = call.has_flag(engine_state, stack, "tail")?;
let df = NuDataFrame::try_from_value(input)?;
let values = if tail {
df.tail(rows, call.head)?
} else {
// if rows is specified, return those rows, otherwise return everything
if rows.is_some() {
df.head(rows, call.head)?
} else {
df.head(Some(df.height()), call.head)?
}
};
let value = Value::list(values, call.head);
Ok(PipelineData::Value(value, None))
}
fn expression_command(call: &Call, input: Value) -> Result<PipelineData, ShellError> {
let expr = NuExpression::try_from_value(input)?;
let value = expr.to_value(call.head)?;
Ok(PipelineData::Value(value, None))
}
#[cfg(test)]
mod test {
use super::super::super::expressions::ExprCol;
use super::super::super::test_dataframe::test_dataframe;
use super::*;
#[test]
fn test_examples_dataframe_input() {
test_dataframe(vec![Box::new(ToNu {})])
}
#[test]
fn test_examples_expression_input() {
test_dataframe(vec![Box::new(ToNu {}), Box::new(ExprCol {})])
}
}

79
crates/nu-cmd-dataframe/src/dataframe/eager/to_parquet.rs Normal file
View File

@ -0,0 +1,79 @@
use crate::dataframe::values::NuDataFrame;
use nu_engine::command_prelude::*;
use polars::prelude::ParquetWriter;
use std::{fs::File, path::PathBuf};
#[derive(Clone)]
pub struct ToParquet;
impl Command for ToParquet {
fn name(&self) -> &str {
"dfr to-parquet"
}
fn usage(&self) -> &str {
"Saves dataframe to parquet file."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("file", SyntaxShape::Filepath, "file path to save dataframe")
.input_output_type(Type::Custom("dataframe".into()), Type::Any)
.category(Category::Custom("dataframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Saves dataframe to parquet file",
example: "[[a b]; [1 2] [3 4]] | dfr into-df | dfr to-parquet test.parquet",
result: None,
}]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
command(engine_state, stack, call, input)
}
}
fn command(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let file_name: Spanned<PathBuf> = call.req(engine_state, stack, 0)?;
let mut df = NuDataFrame::try_from_pipeline(input, call.head)?;
let file = File::create(&file_name.item).map_err(|e| ShellError::GenericError {
error: "Error with file name".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
ParquetWriter::new(file)
.finish(df.as_mut())
.map_err(|e| ShellError::GenericError {
error: "Error saving file".into(),
msg: e.to_string(),
span: Some(file_name.span),
help: None,
inner: vec![],
})?;
let file_value = Value::string(format!("saved {:?}", &file_name.item), file_name.span);
Ok(PipelineData::Value(
Value::list(vec![file_value], call.head),
None,
))
}

203
crates/nu-cmd-dataframe/src/dataframe/eager/with_column.rs Normal file
View File

@ -0,0 +1,203 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression, NuLazyFrame};
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct WithColumn;
impl Command for WithColumn {
fn name(&self) -> &str {
"dfr with-column"
}
fn usage(&self) -> &str {
"Adds a series to the dataframe."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.named("name", SyntaxShape::String, "new column name", Some('n'))
.rest(
"series or expressions",
SyntaxShape::Any,
"series to be added or expressions used to define the new columns",
)
.input_output_type(
Type::Custom("dataframe".into()),
Type::Custom("dataframe".into()),
)
.category(Category::Custom("dataframe or lazyframe".into()))
}
fn examples(&self) -> Vec<Example> {
vec![
Example {
description: "Adds a series to the dataframe",
example: r#"[[a b]; [1 2] [3 4]]
| dfr into-df
| dfr with-column ([5 6] | dfr into-df) --name c"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
Column::new(
"c".to_string(),
vec![Value::test_int(5), Value::test_int(6)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Adds a series to the dataframe",
example: r#"[[a b]; [1 2] [3 4]]
| dfr into-lazy
| dfr with-column [
((dfr col a) * 2 | dfr as "c")
((dfr col a) * 3 | dfr as "d")
]
| dfr collect"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_int(1), Value::test_int(3)],
),
Column::new(
"b".to_string(),
vec![Value::test_int(2), Value::test_int(4)],
),
Column::new(
"c".to_string(),
vec![Value::test_int(2), Value::test_int(6)],
),
Column::new(
"d".to_string(),
vec![Value::test_int(3), Value::test_int(9)],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value = input.into_value(call.head);
if NuLazyFrame::can_downcast(&value) {
let df = NuLazyFrame::try_from_value(value)?;
command_lazy(engine_state, stack, call, df)
} else if NuDataFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value(value)?;
command_eager(engine_state, stack, call, df)
} else {
Err(ShellError::CantConvert {
to_type: "lazy or eager dataframe".into(),
from_type: value.get_type().to_string(),
span: value.span(),
help: None,
})
}
}
}
fn command_eager(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
mut df: NuDataFrame,
) -> Result<PipelineData, ShellError> {
let new_column: Value = call.req(engine_state, stack, 0)?;
let column_span = new_column.span();
if NuExpression::can_downcast(&new_column) {
let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
let value = Value::list(vals, call.head);
let expressions = NuExpression::extract_exprs(value)?;
let lazy = NuLazyFrame::new(true, df.lazy().with_columns(&expressions));
let df = lazy.collect(call.head)?;
Ok(PipelineData::Value(df.into_value(call.head), None))
} else {
let mut other = NuDataFrame::try_from_value(new_column)?.as_series(column_span)?;
let name = match call.get_flag::<String>(engine_state, stack, "name")? {
Some(name) => name,
None => other.name().to_string(),
};
let series = other.rename(&name).clone();
df.as_mut()
.with_column(series)
.map_err(|e| ShellError::GenericError {
error: "Error adding column to dataframe".into(),
msg: e.to_string(),
span: Some(column_span),
help: None,
inner: vec![],
})
.map(|df| {
PipelineData::Value(
NuDataFrame::dataframe_into_value(df.clone(), call.head),
None,
)
})
}
}
fn command_lazy(
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
lazy: NuLazyFrame,
) -> Result<PipelineData, ShellError> {
let vals: Vec<Value> = call.rest(engine_state, stack, 0)?;
let value = Value::list(vals, call.head);
let expressions = NuExpression::extract_exprs(value)?;
let lazy: NuLazyFrame = lazy.into_polars().with_columns(&expressions).into();
Ok(PipelineData::Value(
NuLazyFrame::into_value(lazy, call.head)?,
None,
))
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::expressions::ExprAlias;
use crate::dataframe::expressions::ExprCol;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(WithColumn {}),
Box::new(ExprAlias {}),
Box::new(ExprCol {}),
])
}
}

86
crates/nu-cmd-dataframe/src/dataframe/expressions/alias.rs Normal file
View File

@ -0,0 +1,86 @@
use crate::dataframe::values::NuExpression;
use nu_engine::command_prelude::*;
#[derive(Clone)]
pub struct ExprAlias;
impl Command for ExprAlias {
fn name(&self) -> &str {
"dfr as"
}
fn usage(&self) -> &str {
"Creates an alias expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"Alias name",
SyntaxShape::String,
"Alias name for the expression",
)
.input_output_type(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
)
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Creates and alias expression",
example: "dfr col a | dfr as new_a | dfr into-nu",
result: {
let record = Value::test_record(record! {
"expr" => Value::test_record(record! {
"expr" => Value::test_string("column"),
"value" => Value::test_string("a"),
}),
"alias" => Value::test_string("new_a"),
});
Some(record)
},
}]
}
fn search_terms(&self) -> Vec<&str> {
vec!["aka", "abbr", "otherwise"]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let alias: String = call.req(engine_state, stack, 0)?;
let expr = NuExpression::try_from_pipeline(input, call.head)?;
let expr: NuExpression = expr.into_polars().alias(alias.as_str()).into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::eager::ToNu;
use crate::dataframe::expressions::ExprCol;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(ExprAlias {}),
Box::new(ExprCol {}),
Box::new(ToNu {}),
])
}
}

78
crates/nu-cmd-dataframe/src/dataframe/expressions/arg_where.rs Normal file
View File

@ -0,0 +1,78 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
use polars::prelude::arg_where;
#[derive(Clone)]
pub struct ExprArgWhere;
impl Command for ExprArgWhere {
fn name(&self) -> &str {
"dfr arg-where"
}
fn usage(&self) -> &str {
"Creates an expression that returns the arguments where expression is true."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required("column name", SyntaxShape::Any, "Expression to evaluate")
.input_output_type(Type::Any, Type::Custom("expression".into()))
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Return a dataframe where the value match the expression",
example: "let df = ([[a b]; [one 1] [two 2] [three 3]] | dfr into-df);
$df | dfr select (dfr arg-where ((dfr col b) >= 2) | dfr as b_arg)",
result: Some(
NuDataFrame::try_from_columns(
vec![Column::new(
"b_arg".to_string(),
vec![Value::test_int(1), Value::test_int(2)],
)],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn search_terms(&self) -> Vec<&str> {
vec!["condition", "match", "if"]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let value: Value = call.req(engine_state, stack, 0)?;
let expr = NuExpression::try_from_value(value)?;
let expr: NuExpression = arg_where(expr.into_polars()).into();
Ok(PipelineData::Value(expr.into_value(call.head), None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::expressions::ExprAlias;
use crate::dataframe::lazy::LazySelect;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(ExprArgWhere {}),
Box::new(ExprAlias {}),
Box::new(LazySelect {}),
])
}
}

68
crates/nu-cmd-dataframe/src/dataframe/expressions/col.rs Normal file
View File

@ -0,0 +1,68 @@
use crate::dataframe::values::NuExpression;
use nu_engine::command_prelude::*;
use polars::prelude::col;
#[derive(Clone)]
pub struct ExprCol;
impl Command for ExprCol {
fn name(&self) -> &str {
"dfr col"
}
fn usage(&self) -> &str {
"Creates a named column expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"column name",
SyntaxShape::String,
"Name of column to be used",
)
.input_output_type(Type::Any, Type::Custom("expression".into()))
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Creates a named column expression and converts it to a nu object",
example: "dfr col a | dfr into-nu",
result: Some(Value::test_record(record! {
"expr" => Value::test_string("column"),
"value" => Value::test_string("a"),
})),
}]
}
fn search_terms(&self) -> Vec<&str> {
vec!["create"]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let name: String = call.req(engine_state, stack, 0)?;
let expr: NuExpression = col(name.as_str()).into();
Ok(PipelineData::Value(expr.into_value(call.head), None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::eager::ToNu;
#[test]
fn test_examples() {
test_dataframe(vec![Box::new(ExprCol {}), Box::new(ToNu {})])
}
}

108
crates/nu-cmd-dataframe/src/dataframe/expressions/concat_str.rs Normal file
View File

@ -0,0 +1,108 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use nu_engine::command_prelude::*;
use polars::prelude::concat_str;
#[derive(Clone)]
pub struct ExprConcatStr;
impl Command for ExprConcatStr {
fn name(&self) -> &str {
"dfr concat-str"
}
fn usage(&self) -> &str {
"Creates a concat string expression."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"separator",
SyntaxShape::String,
"Separator used during the concatenation",
)
.required(
"concat expressions",
SyntaxShape::List(Box::new(SyntaxShape::Any)),
"Expression(s) that define the string concatenation",
)
.input_output_type(Type::Any, Type::Custom("expression".into()))
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
vec![Example {
description: "Creates a concat string expression",
example: r#"let df = ([[a b c]; [one two 1] [three four 2]] | dfr into-df);
$df | dfr with-column ((dfr concat-str "-" [(dfr col a) (dfr col b) ((dfr col c) * 2)]) | dfr as concat)"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new(
"a".to_string(),
vec![Value::test_string("one"), Value::test_string("three")],
),
Column::new(
"b".to_string(),
vec![Value::test_string("two"), Value::test_string("four")],
),
Column::new(
"c".to_string(),
vec![Value::test_int(1), Value::test_int(2)],
),
Column::new(
"concat".to_string(),
vec![
Value::test_string("one-two-2"),
Value::test_string("three-four-4"),
],
),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
}]
}
fn search_terms(&self) -> Vec<&str> {
vec!["join", "connect", "update"]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
_input: PipelineData,
) -> Result<PipelineData, ShellError> {
let separator: String = call.req(engine_state, stack, 0)?;
let value: Value = call.req(engine_state, stack, 1)?;
let expressions = NuExpression::extract_exprs(value)?;
let expr: NuExpression = concat_str(expressions, &separator, false).into();
Ok(PipelineData::Value(expr.into_value(call.head), None))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::eager::WithColumn;
use crate::dataframe::expressions::alias::ExprAlias;
use crate::dataframe::expressions::col::ExprCol;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(ExprConcatStr {}),
Box::new(ExprAlias {}),
Box::new(ExprCol {}),
Box::new(WithColumn {}),
])
}
}

170
crates/nu-cmd-dataframe/src/dataframe/expressions/datepart.rs Normal file
View File

@ -0,0 +1,170 @@
use crate::dataframe::values::{Column, NuDataFrame, NuExpression};
use chrono::{DateTime, FixedOffset};
use nu_engine::command_prelude::*;
use polars::{
datatypes::{DataType, TimeUnit},
prelude::NamedFrom,
series::Series,
};
#[derive(Clone)]
pub struct ExprDatePart;
impl Command for ExprDatePart {
fn name(&self) -> &str {
"dfr datepart"
}
fn usage(&self) -> &str {
"Creates an expression for capturing the specified datepart in a column."
}
fn signature(&self) -> Signature {
Signature::build(self.name())
.required(
"Datepart name",
SyntaxShape::String,
"Part of the date to capture. Possible values are year, quarter, month, week, weekday, day, hour, minute, second, millisecond, microsecond, nanosecond",
)
.input_output_type(
Type::Custom("expression".into()),
Type::Custom("expression".into()),
)
.category(Category::Custom("expression".into()))
}
fn examples(&self) -> Vec<Example> {
let dt = DateTime::<FixedOffset>::parse_from_str(
"2021-12-30T01:02:03.123456789 +0000",
"%Y-%m-%dT%H:%M:%S.%9f %z",
)
.expect("date calculation should not fail in test");
vec![
Example {
description: "Creates an expression to capture the year date part",
example: r#"[["2021-12-30T01:02:03.123456789"]] | dfr into-df | dfr as-datetime "%Y-%m-%dT%H:%M:%S.%9f" | dfr with-column [(dfr col datetime | dfr datepart year | dfr as datetime_year )]"#,
result: Some(
NuDataFrame::try_from_columns(
vec![
Column::new("datetime".to_string(), vec![Value::test_date(dt)]),
Column::new("datetime_year".to_string(), vec![Value::test_int(2021)]),
],
None,
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
Example {
description: "Creates an expression to capture multiple date parts",
example: r#"[["2021-12-30T01:02:03.123456789"]] | dfr into-df | dfr as-datetime "%Y-%m-%dT%H:%M:%S.%9f" |
dfr with-column [ (dfr col datetime | dfr datepart year | dfr as datetime_year ),
(dfr col datetime | dfr datepart month | dfr as datetime_month ),
(dfr col datetime | dfr datepart day | dfr as datetime_day ),
(dfr col datetime | dfr datepart hour | dfr as datetime_hour ),
(dfr col datetime | dfr datepart minute | dfr as datetime_minute ),
(dfr col datetime | dfr datepart second | dfr as datetime_second ),
(dfr col datetime | dfr datepart nanosecond | dfr as datetime_ns ) ]"#,
result: Some(
NuDataFrame::try_from_series(
vec![
Series::new("datetime", &[dt.timestamp_nanos_opt()])
.cast(&DataType::Datetime(TimeUnit::Nanoseconds, None))
.expect("Error casting to datetime type"),
Series::new("datetime_year", &[2021_i64]), // i32 was coerced to i64
Series::new("datetime_month", &[12_i8]),
Series::new("datetime_day", &[30_i8]),
Series::new("datetime_hour", &[1_i8]),
Series::new("datetime_minute", &[2_i8]),
Series::new("datetime_second", &[3_i8]),
Series::new("datetime_ns", &[123456789_i64]), // i32 was coerced to i64
],
Span::test_data(),
)
.expect("simple df for test should not fail")
.into_value(Span::test_data()),
),
},
]
}
fn search_terms(&self) -> Vec<&str> {
vec![
"year",
"month",
"week",
"weekday",
"quarter",
"day",
"hour",
"minute",
"second",
"millisecond",
"microsecond",
"nanosecond",
]
}
fn run(
&self,
engine_state: &EngineState,
stack: &mut Stack,
call: &Call,
input: PipelineData,
) -> Result<PipelineData, ShellError> {
let part: Spanned<String> = call.req(engine_state, stack, 0)?;
let expr = NuExpression::try_from_pipeline(input, call.head)?;
let expr_dt = expr.into_polars().dt();
let expr = match part.item.as_str() {
"year" => expr_dt.year(),
"quarter" => expr_dt.quarter(),
"month" => expr_dt.month(),
"week" => expr_dt.week(),
"day" => expr_dt.day(),
"hour" => expr_dt.hour(),
"minute" => expr_dt.minute(),
"second" => expr_dt.second(),
"millisecond" => expr_dt.millisecond(),
"microsecond" => expr_dt.microsecond(),
"nanosecond" => expr_dt.nanosecond(),
_ => {
return Err(ShellError::UnsupportedInput {
msg: format!("{} is not a valid datepart, expected one of year, month, day, hour, minute, second, millisecond, microsecond, nanosecond", part.item),
input: "value originates from here".to_string(),
msg_span: call.head,
input_span: part.span,
});
}
}.into();
Ok(PipelineData::Value(
NuExpression::into_value(expr, call.head),
None,
))
}
}
#[cfg(test)]
mod test {
use super::super::super::test_dataframe::test_dataframe;
use super::*;
use crate::dataframe::eager::ToNu;
use crate::dataframe::eager::WithColumn;
use crate::dataframe::expressions::ExprAlias;
use crate::dataframe::expressions::ExprCol;
use crate::dataframe::series::AsDateTime;
#[test]
fn test_examples() {
test_dataframe(vec![
Box::new(ExprDatePart {}),
Box::new(ExprCol {}),
Box::new(ToNu {}),
Box::new(AsDateTime {}),
Box::new(WithColumn {}),
Box::new(ExprAlias {}),
])
}
}

Some files were not shown because too many files have changed in this diff Show More