Add possibility to declare optional parameters and switch flags (#2966)

* Add possibility to declare optional parameters and switch flags

With this commit applied it is now possible to specify optional parameters and flags
as switches. This PR **only** makes guarantees about **parsing** optional flags and
switches correctly. This PR **does not guarantee flawless functionality** of
optional parameters / switches within scripts.
functionality within scripts. Example:

test.nu
```shell
def my_command [
    opt_param?
    opt_param2?: int
    --switch
] {echo hi nushell}
```

```shell
> source test.nu
> my_command -h
───┬─────────
 0 │ hi
 1 │ nushell
───┴─────────
Usage:
  > my_command <mandatory_param> (opt_param) (opt_param2) {flags}

Parameters:
  <mandatory_param>
  (opt_param)
  (opt_param2)

Flags:
  -h, --help: Display this help message
  --switch
  --opt_flag <any>
```

* Update def docs

crates/nu-parser/src/parse/def/param_flag_list.rs

@@ -1,12 +1,15 @@
-///This module contains functions to parse the parameter and flag list (signature) of a
+///This module contains functions to parse the parameter and flag list (signature)
-///definition
 ///Such a signature can be of the following format:
-/// [ (parameter | flag | <eol>)* ]
+/// [ (parameter | flag | rest_param | <eol>)* ]
 ///Where
 ///parameter is:
-///    name (<:> type)? (<,> | <eol> | (#Comment <eol>))?
+///    name (<:> type)? (<?>)? item_end
 ///flag is:
-///    --name (-shortform)? (<:> type)? (<,> | <eol> | (#Comment <eol>))?
+///    --name (-shortform)? (<:> type)? item_end
+///rest is:
+///    ...rest (<:> type)? item_end
+///item_end:
+///    (<,>)? (#Comment)? (<eol>)?
 ///
 use log::debug;
 
@@ -49,7 +52,7 @@ pub fn parse_signature(
 
     //After normal lexing, tokens also need to be split on ',' and ':'
     //TODO this could probably be all done in a specialized lexing function
-    let tokens = lex_split_baseline_tokens_on(tokens, &[',', ':']);
+    let tokens = lex_split_baseline_tokens_on(tokens, &[',', ':', '?']);
     let tokens = lex_split_shortflag_from_longflag(tokens);
     debug!("Tokens are {:?}", tokens);
 
@@ -100,26 +103,43 @@ fn parse_parameter(
     }
 
     let mut err: Option<ParseError> = None;
-    //1 because name = tokens[0]
+    let mut i = 0;
-    let mut i = 1;
+    let mut type_ = SyntaxShape::Any;
+    let mut comment = None;
+    let mut optional = false;
 
     let (name, error) = parse_param_name(&tokens[0]);
+    i += 1;
     err = err.or(error);
 
-    let (type_, advanced_by, error) = parse_optional_type(&tokens[i..]);
+    if i < tokens.len() {
-    let type_ = type_.unwrap_or(SyntaxShape::Any);
+        let (parsed_opt_modifier, advanced_by) =
-    err = err.or(error);
+            parse_optional_parameter_optional_modifier(&tokens[i]);
-    i += advanced_by;
+        optional = parsed_opt_modifier;
+        i += advanced_by;
+    }
 
-    let (comment_text, advanced_by, error) = parse_signature_item_end(&tokens[i..]);
+    if i < tokens.len() {
-    i += advanced_by;
+        let (parsed_type_, advanced_by, error) = parse_optional_type(&tokens[i..]);
-    err = err.or(error);
+        type_ = parsed_type_.unwrap_or(SyntaxShape::Any);
+        err = err.or(error);
+        i += advanced_by;
+    }
 
-    let parameter = Parameter::new(
+    if i < tokens.len() {
-        PositionalType::mandatory(&name.item, type_),
+        let (comment_text, advanced_by, error) = parse_signature_item_end(&tokens[i..]);
-        comment_text,
+        comment = comment_text;
-        name.span,
+        i += advanced_by;
-    );
+        err = err.or(error);
+    }
+
+    let pos_type = if optional {
+        PositionalType::optional(&name.item, type_)
+    } else {
+        PositionalType::mandatory(&name.item, type_)
+    };
+
+    let parameter = Parameter::new(pos_type, comment, name.span);
 
     debug!(
         "Parsed parameter: {} with shape {:?}",
@@ -143,32 +163,47 @@ fn parse_flag(
     }
 
     let mut err: Option<ParseError> = None;
-    //1 because name = tokens[0]
+    let mut i = 0;
-    let mut i = 1;
+    let mut shortform = None;
+    let mut type_ = None;
+    let mut comment = None;
 
     let (name, error) = parse_flag_name(&tokens[0]);
     err = err.or(error);
+    i += 1;
 
-    let (shortform, advanced_by, error) = parse_flag_optional_shortform(&tokens[i..]);
+    if i < tokens.len() {
-    i += advanced_by;
+        let (parsed_shortform, advanced_by, error) = parse_flag_optional_shortform(&tokens[i..]);
-    err = err.or(error);
+        shortform = parsed_shortform;
+        i += advanced_by;
+        err = err.or(error);
+    }
 
-    let (type_, advanced_by, error) = parse_optional_type(&tokens[i..]);
+    if i < tokens.len() {
-    let type_ = type_.unwrap_or(SyntaxShape::Any);
+        let (parsed_type, advanced_by, error) = parse_optional_type(&tokens[i..]);
-    err = err.or(error);
+        type_ = parsed_type;
-    i += advanced_by;
+        i += advanced_by;
+        err = err.or(error);
+    }
 
-    let (comment, advanced_by, error) = parse_signature_item_end(&tokens[i..]);
+    if i < tokens.len() {
-    i += advanced_by;
+        let (parsed_comment, advanced_by, error) = parse_signature_item_end(&tokens[i..]);
-    err = err.or(error);
+        comment = parsed_comment;
+        i += advanced_by;
+        err = err.or(error);
+    }
 
-    //TODO Fixup span
+    //If no type is given, the flag is a switch. Otherwise its optional
-    let flag = Flag::new(
+    //Example:
-        name.item.clone(),
+    //--verbose(-v) # Switch
-        NamedType::Optional(shortform, type_),
+    //--output(-o): path # Optional flag
-        comment,
+    let named_type = if let Some(shape) = type_ {
-        name.span,
+        NamedType::Optional(shortform, shape)
-    );
+    } else {
+        NamedType::Switch(shortform)
+    };
+
+    let flag = Flag::new(name.item.clone(), named_type, comment, name.span);
 
     debug!("Parsed flag: {:?}", flag);
     (flag, i, err)
@@ -195,19 +230,25 @@ fn parse_rest(
 
     let mut err = None;
     let mut i = 0;
+    let mut type_ = SyntaxShape::Any;
+    let mut comment = "".to_string();
 
     let error = parse_rest_name(&tokens[i]);
     err = err.or(error);
     i += 1;
 
-    let (type_, advanced_by, error) = parse_optional_type(&tokens[i..]);
+    if i < tokens.len() {
-    err = err.or(error);
+        let (parsed_type, advanced_by, error) = parse_optional_type(&tokens[i..]);
-    i += advanced_by;
+        err = err.or(error);
-    let type_ = type_.unwrap_or(SyntaxShape::Any);
+        i += advanced_by;
+        type_ = parsed_type.unwrap_or(SyntaxShape::Any);
+    }
 
-    let (comment, advanced_by) = parse_optional_comment(&tokens[i..]);
+    if i < tokens.len() {
-    i += advanced_by;
+        let (parsed_comment, advanced_by) = parse_optional_comment(&tokens[i..]);
-    let comment = comment.unwrap_or_else(|| "".to_string());
+        i += advanced_by;
+        comment = parsed_comment.unwrap_or_else(|| "".to_string());
+    }
 
     return (Some((type_, comment)), i, err);
 
@@ -320,8 +361,20 @@ fn parse_optional_type(tokens: &[Token]) -> (Option<SyntaxShape>, usize, Option<
     (type_, i, err)
 }
 
+///Parse token if it is a modifier to make
+fn parse_optional_parameter_optional_modifier(token: &Token) -> (bool, usize) {
+    fn is_questionmark(token: &Token) -> bool {
+        is_baseline_token_matching(token, "?")
+    }
+    if is_questionmark(token) {
+        (true, 1)
+    } else {
+        (false, 0)
+    }
+}
+
 ///Parses the end of a flag or a parameter
-///    ((<,> | <eol>) | (#Comment <eol>)
+///   (<,>)? (#Comment)? (<eol>)?
 fn parse_signature_item_end(tokens: &[Token]) -> (Option<String>, usize, Option<ParseError>) {
     if tokens.is_empty() {
         //If no more tokens, parameter/flag doesn't need ',' or comment to be properly finished
@@ -546,7 +599,7 @@ fn lex_split_shortflag_from_longflag(tokens: Vec<Token>) -> Vec<Token> {
     }
     result
 }
-//Currently the lexer does not split baselines on ',' ':'
+//Currently the lexer does not split baselines on ',' ':' '?'
 //The parameter list requires this. Therefore here is a hacky method doing this.
 fn lex_split_baseline_tokens_on(
     tokens: Vec<Token>,
@@ -672,14 +725,14 @@ mod tests {
     #[test]
     fn simple_def_with_params() {
         let name = "my_func";
-        let sign = "[param1:int, param2:string]";
+        let sign = "[param1?: int, param2: string]";
         let (sign, err) = parse_signature(name, &sign.to_string().spanned(Span::new(0, 27)));
         assert!(err.is_none());
         assert_eq!(
             sign.positional,
             vec![
                 (
-                    PositionalType::Mandatory("param1".into(), SyntaxShape::Int),
+                    PositionalType::Optional("param1".into(), SyntaxShape::Int),
                     "".into()
                 ),
                 (
@@ -690,6 +743,27 @@ mod tests {
         );
     }
 
+    #[test]
+    fn simple_def_with_optional_param_without_type() {
+        let name = "my_func";
+        let sign = "[param1 ?, param2?]";
+        let (sign, err) = parse_signature(name, &sign.to_string().spanned(Span::new(0, 27)));
+        assert!(err.is_none());
+        assert_eq!(
+            sign.positional,
+            vec![
+                (
+                    PositionalType::Optional("param1".into(), SyntaxShape::Any),
+                    "".into()
+                ),
+                (
+                    PositionalType::Optional("param2".into(), SyntaxShape::Any),
+                    "".into()
+                ),
+            ]
+        );
+    }
+
     #[test]
     fn simple_def_with_params_with_comment() {
         let name = "my_func";
@@ -802,8 +876,8 @@ mod tests {
         let sign = "[
         --list (-l) : path  # First flag
         --verbose : number # Second flag
+        --all(-a) # My switch
         ]";
-        // --all(-a) # My switch
         let (sign, err) = parse_signature(name, &sign.to_string().spanned_unknown());
         assert!(err.is_none());
         assert_signature_has_flag(
@@ -818,12 +892,7 @@ mod tests {
             NamedType::Optional(None, SyntaxShape::Number),
             "Second flag",
         );
-        // assert_signature_has_flag(
+        assert_signature_has_flag(&sign, "all", NamedType::Switch(Some('a')), "My switch");
-        //     &sign,
-        //     "verbose",
-        //     NamedType::Switch(Some('a')),
-        //     "Second flag",
-        // );
     }
 
     #[test]
@@ -835,6 +904,7 @@ mod tests {
         --verbose # Second flag
         param3 : number,
         --flag3 # Third flag
+        param4 ?: table # Optional Param
         ]";
         let (sign, err) = parse_signature(name, &sign.to_string().spanned_unknown());
         assert!(err.is_none());
@@ -844,25 +914,10 @@ mod tests {
             NamedType::Optional(Some('l'), SyntaxShape::FilePath),
             "First flag",
         );
-        assert_signature_has_flag(
+        assert_signature_has_flag(&sign, "verbose", NamedType::Switch(None), "Second flag");
-            &sign,
+        assert_signature_has_flag(&sign, "flag3", NamedType::Switch(None), "Third flag");
-            "verbose",
-            NamedType::Optional(None, SyntaxShape::Any),
-            "Second flag",
-        );
-        assert_signature_has_flag(
-            &sign,
-            "flag3",
-            NamedType::Optional(None, SyntaxShape::Any),
-            "Third flag",
-        );
         assert_eq!(
             sign.positional,
-            // --list (-l) : path  # First flag
-            // param1, param2:table # Param2 Doc
-            // --verbose # Second flag
-            // param3 : number,
-            // --flag3 # Third flag
             vec![
                 (
                     PositionalType::Mandatory("param1".into(), SyntaxShape::Any),
@@ -876,6 +931,10 @@ mod tests {
                     PositionalType::Mandatory("param3".into(), SyntaxShape::Number),
                     "".into()
                 ),
+                (
+                    PositionalType::Optional("param4".into(), SyntaxShape::Table),
+                    "Optional Param".into()
+                ),
             ]
         );
     }
@@ -888,12 +947,7 @@ mod tests {
             ]";
         let (sign, err) = parse_signature(name, &sign.to_string().spanned_unknown());
         assert!(err.is_none());
-        assert_signature_has_flag(
+        assert_signature_has_flag(&sign, "force", NamedType::Switch(Some('f')), "");
-            &sign,
-            "force",
-            NamedType::Optional(Some('f'), SyntaxShape::Any),
-            "",
-        );
         assert_eq!(
             sign.positional,
             // --list (-l) : path  # First flag
@@ -1003,7 +1057,7 @@ mod tests {
         assert_signature_has_flag(
             &sign,
             "xxx",
-            NamedType::Optional(Some('x'), SyntaxShape::Any),
+            NamedType::Switch(Some('x')),
             "The all powerful x flag",
         );
         assert_signature_has_flag(

docs/commands/def.md

@@ -4,14 +4,70 @@ Use `def` to create a custom command.
 
 ## Examples
 
-```
+```shell
 > def my_command [] { echo hi nu }
 > my_command
 hi nu
 ```
 
-```
+```shell
 > def my_command [adjective: string, num: int] { echo $adjective $num meet nu }
 > my_command nice 2
 nice 2 meet nu
 ```
+
+```shell
+def my_cookie_daemon [
+    in: path             # Specify where the cookie daemon shall look for cookies :p
+    ...rest: path        # Other places to consider for cookie supplies
+    --output (-o): path  # Where to store leftovers
+    --verbose
+] {
+    echo $in $rest | each { eat $it }
+    ...
+}
+my_cookie_daemon /home/bob /home/alice --output /home/mallory
+```
+
+Further (and non trivial) examples can be found in our [nushell scripts repo](https://github.com/nushell/nu_scripts)
+
+## Syntax
+
+The syntax of the def command is as follows.
+`def <name> <signature> <block>`
+
+The signature is a list of parameters flags and at maximum one rest argument. You can specify the type of each of them by appending `: <type>`.
+Example:
+```shell
+def cmd [
+parameter: string
+--flag: int
+...rest: path
+] { ... }
+```
+
+It is possible to comment them by appending `# Comment text`!
+Example
+```shell
+def cmd [
+parameter # Paramter Comment
+--flag: int # Flag comment
+...rest: path # Rest comment
+] { ... }
+```
+
+Flags can have a single character shorthand form. For example `--output` is often abbreviated by `-o`. You can declare a shorthand by writing `(-<shorthand>)` after the flag name.
+Example
+```shell
+def cmd [
+--flag(-f): int # Flag comment
+] { ... }
+```
+
+You can make a parameter optional by adding `?` to its name. Optional parameters do not need to be passed.
+(TODO Handling optional parameters in scripts is WIP. Please don't expect it to work seamlessly)
+```shell
+def cmd [
+parameter?: path # Optional parameter
+] { ... }
+```