From 37f7a361236c11aa64fd210e982d952d9211b9dc Mon Sep 17 00:00:00 2001 From: Fernando Herrera Date: Sat, 30 Oct 2021 14:21:59 +0100 Subject: [PATCH] syntax serializers --- crates/nu-plugin/build.rs | 2 +- crates/nu-plugin/schema/plugin.capnp | 102 ++++++ crates/nu-plugin/schema/value.capnp | 57 ---- crates/nu-plugin/src/lib.rs | 4 +- crates/nu-plugin/src/serializers/call.rs | 11 +- crates/nu-plugin/src/serializers/callinfo.rs | 2 +- crates/nu-plugin/src/serializers/mod.rs | 1 + crates/nu-plugin/src/serializers/signature.rs | 312 ++++++++++++++++++ crates/nu-plugin/src/serializers/value.rs | 6 +- 9 files changed, 427 insertions(+), 70 deletions(-) create mode 100644 crates/nu-plugin/schema/plugin.capnp delete mode 100644 crates/nu-plugin/schema/value.capnp create mode 100644 crates/nu-plugin/src/serializers/signature.rs diff --git a/crates/nu-plugin/build.rs b/crates/nu-plugin/build.rs index 40757c1621..a88b401e34 100644 --- a/crates/nu-plugin/build.rs +++ b/crates/nu-plugin/build.rs @@ -1,7 +1,7 @@ fn main() { capnpc::CompilerCommand::new() .src_prefix("schema") - .file("schema/value.capnp") + .file("schema/plugin.capnp") .run() .expect("compiling schema"); } diff --git a/crates/nu-plugin/schema/plugin.capnp b/crates/nu-plugin/schema/plugin.capnp new file mode 100644 index 0000000000..db29f5208c --- /dev/null +++ b/crates/nu-plugin/schema/plugin.capnp @@ -0,0 +1,102 @@ +@0xb299d30dc02d72bc; +# Schema representing all the structs that are used to comunicate with +# the plugins. +# This schema, together with the command capnp proto is used to generate +# the rust file that defines the serialization/deserialization objects +# required to comunicate with the plugins created for nushell + +# Generic structs used as helpers for the encoding +struct Option(Value) { + union { + none @0 :Void; + some @1 :Value; + } +} + +struct Map(Key, Value) { + struct Entry { + key @0 :Key; + value @1 :Value; + } + entries @0 :List(Entry); +} + +# Main plugin structures +struct Span { + start @0 :UInt64; + end @1 :UInt64; +} + +struct Value { + span @0: Span; + + union { + void @1 :Void; + bool @2 :Bool; + int @3 :Int64; + float @4 :Float64; + string @5 :Text; + list @6 :List(Value); + } +} + +struct Signature { + name @0 :Text; + usage @1 :Text; + extraUsage @2 :Text; + requiredPositional @3 :List(Argument); + optionalPositional @4 :List(Argument); + rest @5 :Option(Argument); + named @6 :List(Flag); + isFilter @7 :Bool; +} + +struct Flag { + long @0 :Text; + short @1 :Option(Text); + arg @2 :Shape; + required @3 :Bool; + desc @4 :Text; +} + +struct Argument { + name @0 :Text; + desc @1 :Text; + shape @2 :Shape; +} + +# If we require more complex signatures for the plugins this could be +# changed to a union +enum Shape { + none @0; + any @1; + string @2; + number @3; + int @4; + boolean @5; +} + +struct Expression { + union { + garbage @0 :Void; + bool @1 :Bool; + int @2 :Int64; + float @3 :Float64; + string @4 :Text; + list @5 :List(Expression); + # The expression list can be exteded based on the user need + # If a plugin requires something from the expression object, it + # will need to be added to this list + } +} + +struct Call { + head @0: Span; + positional @1 :List(Expression); + named @2 :Map(Text, Option(Expression)); +} + +struct CallInfo { + call @0: Call; + input @1: Value; +} diff --git a/crates/nu-plugin/schema/value.capnp b/crates/nu-plugin/schema/value.capnp deleted file mode 100644 index 36077f25af..0000000000 --- a/crates/nu-plugin/schema/value.capnp +++ /dev/null @@ -1,57 +0,0 @@ -@0xb299d30dc02d72bc; - -# Generic structs used as helpers for the encoding -struct Option(Value) { - union { - none @0 :Void; - some @1 :Value; - } -} - -struct Map(Key, Value) { - struct Entry { - key @0 :Key; - value @1 :Value; - } - entries @0 :List(Entry); -} - -struct Span { - start @0 :UInt64; - end @1 :UInt64; -} - -struct Value { - span @0: Span; - - union { - void @1 :Void; - bool @2 :Bool; - int @3 :Int64; - float @4 :Float64; - string @5 :Text; - list @6 :List(Value); - } -} - -struct Expression { - union { - garbage @0 :Void; - bool @1 :Bool; - int @2 :Int64; - float @3 :Float64; - string @4 :Text; - list @5 :List(Expression); - } -} - -struct Call { - head @0: Span; - positional @1 :List(Expression); - named @2 :Map(Text, Option(Expression)); -} - -struct CallInfo { - call @0: Call; - input @1: Value; -} diff --git a/crates/nu-plugin/src/lib.rs b/crates/nu-plugin/src/lib.rs index 0588db667e..c4d1a54347 100644 --- a/crates/nu-plugin/src/lib.rs +++ b/crates/nu-plugin/src/lib.rs @@ -1,6 +1,6 @@ pub mod plugin; pub mod serializers; -pub mod value_capnp { - include!(concat!(env!("OUT_DIR"), "/value_capnp.rs")); +pub mod plugin_capnp { + include!(concat!(env!("OUT_DIR"), "/plugin_capnp.rs")); } diff --git a/crates/nu-plugin/src/serializers/call.rs b/crates/nu-plugin/src/serializers/call.rs index 7c9e4c9038..36afc70cd1 100644 --- a/crates/nu-plugin/src/serializers/call.rs +++ b/crates/nu-plugin/src/serializers/call.rs @@ -1,4 +1,4 @@ -use crate::value_capnp::{call, expression, option}; +use crate::plugin_capnp::{call, expression, option}; use capnp::serialize_packed; use nu_protocol::{ ast::{Call, Expr, Expression}, @@ -69,7 +69,7 @@ fn serialize_expression(expression: &Expression, mut builder: expression::Builde Expr::Bool(val) => builder.set_bool(*val), Expr::Int(val) => builder.set_int(*val), Expr::Float(val) => builder.set_float(*val), - Expr::String(val) => builder.set_string(&val), + Expr::String(val) => builder.set_string(val), Expr::List(values) => { let mut list_builder = builder.reborrow().init_list(values.len() as u32); for (index, expression) in values.iter().enumerate() { @@ -130,10 +130,9 @@ fn deserialize_positionals( .collect() } -fn deserialize_named( - span: Span, - reader: call::Reader, -) -> Result, Option)>, ShellError> { +type NamedList = Vec<(Spanned, Option)>; + +fn deserialize_named(span: Span, reader: call::Reader) -> Result { let named_reader = reader .get_named() .map_err(|e| ShellError::DecodingError(e.to_string()))?; diff --git a/crates/nu-plugin/src/serializers/callinfo.rs b/crates/nu-plugin/src/serializers/callinfo.rs index ecd0cff2be..5023d666a4 100644 --- a/crates/nu-plugin/src/serializers/callinfo.rs +++ b/crates/nu-plugin/src/serializers/callinfo.rs @@ -1,5 +1,5 @@ use super::{call, value}; -use crate::value_capnp::call_info; +use crate::plugin_capnp::call_info; use capnp::serialize_packed; use nu_protocol::{ast::Call, ShellError, Value}; diff --git a/crates/nu-plugin/src/serializers/mod.rs b/crates/nu-plugin/src/serializers/mod.rs index 3c4ebb40b7..4bd1691b69 100644 --- a/crates/nu-plugin/src/serializers/mod.rs +++ b/crates/nu-plugin/src/serializers/mod.rs @@ -1,3 +1,4 @@ pub mod call; pub mod callinfo; +pub mod signature; pub mod value; diff --git a/crates/nu-plugin/src/serializers/signature.rs b/crates/nu-plugin/src/serializers/signature.rs new file mode 100644 index 0000000000..cbff82580e --- /dev/null +++ b/crates/nu-plugin/src/serializers/signature.rs @@ -0,0 +1,312 @@ +use crate::plugin_capnp::{argument, flag, option, signature, Shape}; +use capnp::serialize_packed; +use nu_protocol::{Flag, PositionalArg, ShellError, Signature, SyntaxShape}; + +pub fn write_buffer( + signature: &Signature, + writer: &mut impl std::io::Write, +) -> Result<(), ShellError> { + let mut message = ::capnp::message::Builder::new_default(); + + let builder = message.init_root::(); + + serialize_signature(signature, builder); + + serialize_packed::write_message(writer, &message) + .map_err(|e| ShellError::EncodingError(e.to_string())) +} + +pub(crate) fn serialize_signature(signature: &Signature, mut builder: signature::Builder) { + builder.set_name(signature.name.as_str()); + builder.set_usage(signature.usage.as_str()); + builder.set_extra_usage(signature.extra_usage.as_str()); + builder.set_is_filter(signature.is_filter); + + // Serializing list of required arguments + let mut required_list = builder + .reborrow() + .init_required_positional(signature.required_positional.len() as u32); + + for (index, arg) in signature.required_positional.iter().enumerate() { + let inner_builder = required_list.reborrow().get(index as u32); + serialize_argument(arg, inner_builder) + } + + // Serializing list of optional arguments + let mut optional_list = builder + .reborrow() + .init_optional_positional(signature.optional_positional.len() as u32); + + for (index, arg) in signature.optional_positional.iter().enumerate() { + let inner_builder = optional_list.reborrow().get(index as u32); + serialize_argument(arg, inner_builder) + } + + // Serializing rest argument + let mut rest_argument = builder.reborrow().init_rest(); + match &signature.rest_positional { + None => rest_argument.set_none(()), + Some(arg) => { + let inner_builder = rest_argument.init_some(); + serialize_argument(arg, inner_builder) + } + } + + // Serializing the named arguments + let mut named_list = builder.reborrow().init_named(signature.named.len() as u32); + + for (index, arg) in signature.named.iter().enumerate() { + let inner_builder = named_list.reborrow().get(index as u32); + serialize_flag(arg, inner_builder) + } +} + +fn serialize_argument(arg: &PositionalArg, mut builder: argument::Builder) { + builder.set_name(arg.name.as_str()); + builder.set_desc(arg.desc.as_str()); + + match arg.shape { + SyntaxShape::Boolean => builder.set_shape(Shape::Boolean), + SyntaxShape::String => builder.set_shape(Shape::String), + SyntaxShape::Int => builder.set_shape(Shape::Int), + SyntaxShape::Number => builder.set_shape(Shape::Number), + _ => builder.set_shape(Shape::Any), + } +} + +fn serialize_flag(arg: &Flag, mut builder: flag::Builder) { + builder.set_long(arg.long.as_str()); + builder.set_required(arg.required); + builder.set_desc(arg.desc.as_str()); + + let mut short_builder = builder.reborrow().init_short(); + match arg.short { + None => short_builder.set_none(()), + Some(val) => { + let mut inner_builder = short_builder.reborrow().initn_some(1); + inner_builder.push_str(format!("{}", val).as_str()); + } + } + + match &arg.arg { + None => builder.set_arg(Shape::None), + Some(shape) => match shape { + SyntaxShape::Boolean => builder.set_arg(Shape::Boolean), + SyntaxShape::String => builder.set_arg(Shape::String), + SyntaxShape::Int => builder.set_arg(Shape::Int), + SyntaxShape::Number => builder.set_arg(Shape::Number), + _ => builder.set_arg(Shape::Any), + }, + } +} + +pub fn read_buffer(reader: &mut impl std::io::BufRead) -> Result { + let message_reader = + serialize_packed::read_message(reader, ::capnp::message::ReaderOptions::new()).unwrap(); + + let reader = message_reader + .get_root::() + .map_err(|e| ShellError::DecodingError(e.to_string()))?; + + deserialize_signature(reader) +} + +pub(crate) fn deserialize_signature(reader: signature::Reader) -> Result { + let name = reader + .get_name() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + let usage = reader + .get_usage() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + let extra_usage = reader + .get_extra_usage() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + let is_filter = reader.get_is_filter(); + + // Deserializing required arguments + let required_list = reader + .get_required_positional() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + + let required_positional = required_list + .iter() + .map(deserialize_argument) + .collect::, ShellError>>()?; + + // Deserializing optional arguments + let optional_list = reader + .get_optional_positional() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + + let optional_positional = optional_list + .iter() + .map(deserialize_argument) + .collect::, ShellError>>()?; + + // Deserializing rest arguments + let rest_option = reader + .get_rest() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + + let rest_positional = match rest_option.which() { + Err(capnp::NotInSchema(_)) => None, + Ok(option::None(())) => None, + Ok(option::Some(rest_reader)) => { + let rest_reader = rest_reader.map_err(|e| ShellError::EncodingError(e.to_string()))?; + Some(deserialize_argument(rest_reader)?) + } + }; + + // Deserializing named arguments + let named_list = reader + .get_named() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + + let named = named_list + .iter() + .map(deserialize_flag) + .collect::, ShellError>>()?; + + Ok(Signature { + name: name.to_string(), + usage: usage.to_string(), + extra_usage: extra_usage.to_string(), + required_positional, + optional_positional, + rest_positional, + named, + is_filter, + creates_scope: false, + }) +} + +fn deserialize_argument(reader: argument::Reader) -> Result { + let name = reader + .get_name() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + + let desc = reader + .get_desc() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + + let shape = reader + .get_shape() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + + let shape = match shape { + Shape::String => SyntaxShape::String, + Shape::Int => SyntaxShape::Int, + Shape::Number => SyntaxShape::Number, + Shape::Boolean => SyntaxShape::Boolean, + Shape::Any => SyntaxShape::Any, + Shape::None => SyntaxShape::Any, + }; + + Ok(PositionalArg { + name: name.to_string(), + desc: desc.to_string(), + shape, + var_id: None, + }) +} + +fn deserialize_flag(reader: flag::Reader) -> Result { + let long = reader + .get_long() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + + let desc = reader + .get_desc() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + + let required = reader.get_required(); + + let short = reader + .get_short() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + + let short = match short.which() { + Err(capnp::NotInSchema(_)) => None, + Ok(option::None(())) => None, + Ok(option::Some(reader)) => { + let reader = reader.map_err(|e| ShellError::EncodingError(e.to_string()))?; + reader.chars().next() + } + }; + + let arg = reader + .get_arg() + .map_err(|e| ShellError::EncodingError(e.to_string()))?; + + let arg = match arg { + Shape::None => None, + Shape::Any => Some(SyntaxShape::Any), + Shape::String => Some(SyntaxShape::String), + Shape::Int => Some(SyntaxShape::Int), + Shape::Number => Some(SyntaxShape::Number), + Shape::Boolean => Some(SyntaxShape::Boolean), + }; + + Ok(Flag { + long: long.to_string(), + short, + arg, + required, + desc: desc.to_string(), + var_id: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use nu_protocol::{Signature, SyntaxShape}; + + #[test] + fn value_round_trip() { + let signature = Signature::build("nu-plugin") + .required("first", SyntaxShape::String, "first required") + .required("second", SyntaxShape::Int, "second required") + .required_named("first_named", SyntaxShape::String, "first named", Some('f')) + .required_named( + "second_named", + SyntaxShape::String, + "second named", + Some('s'), + ) + .rest("remaining", SyntaxShape::Int, "remaining"); + + let mut buffer: Vec = Vec::new(); + write_buffer(&signature, &mut buffer).expect("unable to serialize message"); + let returned_signature = + read_buffer(&mut buffer.as_slice()).expect("unable to deserialize message"); + + assert_eq!(signature.name, returned_signature.name); + assert_eq!(signature.usage, returned_signature.usage); + assert_eq!(signature.extra_usage, returned_signature.extra_usage); + assert_eq!(signature.is_filter, returned_signature.is_filter); + + signature + .required_positional + .iter() + .zip(returned_signature.required_positional.iter()) + .for_each(|(lhs, rhs)| assert_eq!(lhs, rhs)); + + signature + .optional_positional + .iter() + .zip(returned_signature.optional_positional.iter()) + .for_each(|(lhs, rhs)| assert_eq!(lhs, rhs)); + + signature + .named + .iter() + .zip(returned_signature.named.iter()) + .for_each(|(lhs, rhs)| assert_eq!(lhs, rhs)); + + assert_eq!( + signature.rest_positional, + returned_signature.rest_positional, + ); + } +} diff --git a/crates/nu-plugin/src/serializers/value.rs b/crates/nu-plugin/src/serializers/value.rs index c66fe39195..c424a930dc 100644 --- a/crates/nu-plugin/src/serializers/value.rs +++ b/crates/nu-plugin/src/serializers/value.rs @@ -1,4 +1,4 @@ -use crate::value_capnp::value; +use crate::plugin_capnp::value; use capnp::serialize_packed; use nu_protocol::{ShellError, Span, Value}; @@ -35,7 +35,7 @@ pub(crate) fn serialize_value(value: &Value, mut builder: value::Builder) -> Spa *span } Value::String { val, span } => { - builder.set_string(&val); + builder.set_string(val); *span } Value::List { vals, span } => { @@ -92,7 +92,7 @@ pub(crate) fn deserialize_value(reader: value::Reader) -> Result, ShellError>>()?; Ok(Value::List {