// Copyright 2015 Nicholas Allegra (comex).// Licensed under the Apache License, Version 2.0 <https://www.apache.org/licenses/LICENSE-2.0> or// the MIT license <https://opensource.org/licenses/MIT>, at your option. This file may not be// copied, modified, or distributed except according to those terms.//! [`Shlex`] and friends for byte strings.//!//! This is used internally by the [outer module](crate), and may be more//! convenient if you are working with byte slices (`[u8]`) or types that are//! wrappers around bytes, such as [`OsStr`](std::ffi::OsStr)://!//! ```rust//! #[cfg(unix)] {//!     use shlex::bytes::try_quote;//!     use std::ffi::OsStr;//!     use std::os::unix::ffi::OsStrExt;//!//!     // `\x80` is invalid in UTF-8.//!     let os_str = OsStr::from_bytes(b"a\x80b c");//!     assert_eq!(try_quote(os_str.as_bytes()).unwrap(), &b"'a\x80b c'"[..]);//! }//! ```//!//! (On Windows, `OsStr` uses 16 bit wide characters so this will not work.)extern crate alloc;use alloc::vec::Vec;use alloc::borrow::Cow;#[cfg(test)]use alloc::borrow::ToOwned;#[cfg(all(doc, not(doctest)))]use crate::{self as shlex, quoting_warning};use super::QuoteError;

/// An iterator that takes an input byte string and splits it into the words using the same syntax as/// the POSIX shell.pub struct Shlex<'a> {    in_iter: core::slice::Iter<'a, u8>,    /// The number of newlines read so far, plus one.    pub line_no: usize,    /// An input string is erroneous if it ends while inside a quotation or right after an    /// unescaped backslash.  Since Iterator does not have a mechanism to return an error, if that    /// happens, Shlex just throws out the last token, ends the iteration, and sets 'had_error' to    /// true; best to check it after you're done iterating.    pub had_error: bool,}impl<'a> Shlex<'a> {    pub fn new(in_bytes: &'a [u8]) -> Self {        Shlex {            in_iter: in_bytes.iter(),            line_no: 1,            had_error: false,        }    }    fn parse_word(&mut self, mut ch: u8) -> Option<Vec<u8>> {        let mut result: Vec<u8> = Vec::new();        loop {            match ch as char {                '"' => if let Err(()) = self.parse_double(&mut result) {                    self.had_error = true;                    return None;                },                '\'' => if let Err(()) = self.parse_single(&mut result) {                    self.had_error = true;                    return None;                },                '\\' => if let Some(ch2) = self.next_char() {                    if ch2 != b'\n' { result.push(ch2); }                } else {                    self.had_error = true;                    return None;                },                ' ' | '\t' | '\n' => { break; },                _ => { result.push(ch); },            }            if let Some(ch2) = self.next_char() { ch = ch2; } else { break; }        }        Some(result)    }    fn parse_double(&mut self, result: &mut Vec<u8>) -> Result<(), ()> {        loop {            if let Some(ch2) = self.next_char() {                match ch2 as char {                    '\\' => {                        if let Some(ch3) = self.next_char() {                            match ch3 as char {                                // \$ => $                                '$' | '`' | '"' | '\\' => { result.push(ch3); },                                // \<newline> => nothing                                '\n' => {},                                // \x => =x                                _ => { result.push(b'\\'); result.push(ch3); }                            }                        } else {                            return Err(());                        }                    },                    '"' => { return Ok(()); },                    _ => { result.push(ch2); },                }            } else {                return Err(());            }        }    }    fn parse_single(&mut self, result: &mut Vec<u8>) -> Result<(), ()> {        loop {            if let Some(ch2) = self.next_char() {                match ch2 as char {                    '\'' => { return Ok(()); },                    _ => { result.push(ch2); },                }            } else {                return Err(());            }        }    }    fn next_char(&mut self) -> Option<u8> {        let res = self.in_iter.next().copied();        if res == Some(b'\n') { self.line_no += 1; }        res    }}impl Iterator for Shlex<'_> {    type Item = Vec<u8>;    fn next(&mut self) -> Option<Self::Item> {        if let Some(mut ch) = self.next_char() {            // skip initial whitespace            loop {                match ch as char {                    ' ' | '\t' | '\n' => {},                    '#' => {                        while let Some(ch2) = self.next_char() {                            if ch2 as char == '\n' { break; }                        }                    },                    _ => { break; }                }                if let Some(ch2) = self.next_char() { ch = ch2; } else { return None; }            }            self.parse_word(ch)        } else { // no initial character            None        }    }}

/// Convenience function that consumes the whole byte string at once.  Returns None if the input was/// erroneous.pub fn split(in_bytes: &[u8]) -> Option<Vec<Vec<u8>>> {    let mut shl = Shlex::new(in_bytes);    let res = shl.by_ref().collect();    if shl.had_error { None } else { Some(res) }}/// A more configurable interface to quote strings.  If you only want the default settings you can/// use the convenience functions [`try_quote`] and [`try_join`].////// The string equivalent is [`shlex::Quoter`].#[derive(Default, Debug, Clone)]pub struct Quoter {    allow_nul: bool,    // TODO: more options}impl Quoter {    /// Create a new [`Quoter`] with default settings.    #[inline]    pub fn new() -> Self {        Self::default()    }    /// Set whether to allow [nul bytes](quoting_warning#nul-bytes).  By default they are not    /// allowed and will result in an error of [`QuoteError::Nul`].    #[inline]    pub fn allow_nul(mut self, allow: bool) -> Self {        self.allow_nul = allow;        self    }    /// Convenience function that consumes an iterable of words and turns it into a single byte string,    /// quoting words when necessary. Consecutive words will be separated by a single space.    pub fn join<'a, I: IntoIterator<Item = &'a [u8]>>(&self, words: I) -> Result<Vec<u8>, QuoteError> {        Ok(words.into_iter()            .map(|word| self.quote(word))            .collect::<Result<Vec<Cow<[u8]>>, QuoteError>>()?            .join(&b' '))    }    /// Given a single word, return a byte string suitable to encode it as a shell argument.    ///    /// If given valid UTF-8, this will never produce invalid UTF-8. This is because it only    /// ever inserts valid ASCII characters before or after existing ASCII characters (or    /// returns two single quotes if the input was an empty string). It will never modify a    /// multibyte UTF-8 character.    pub fn quote<'a>(&self, mut in_bytes: &'a [u8]) -> Result<Cow<'a, [u8]>, QuoteError> {        if in_bytes.is_empty() {            // Empty string.  Special case that isn't meaningful as only part of a word.            return Ok(b"''"[..].into());        }        if !self.allow_nul && in_bytes.contains(&b'\0') {            return Err(QuoteError::Nul);        }        let mut out: Vec<u8> = Vec::new();        while !in_bytes.is_empty() {            // Pick a quoting strategy for some prefix of the input.  Normally this will cover the            // entire input, but in some case we might need to divide the input into multiple chunks            // that are quoted differently.            let (cur_len, strategy) = quoting_strategy(in_bytes);            if cur_len == in_bytes.len() && strategy == QuotingStrategy::Unquoted && out.is_empty() {                // Entire string can be represented unquoted.  Reuse the allocation.                return Ok(in_bytes.into());            }            let (cur_chunk, rest) = in_bytes.split_at(cur_len);            assert!(rest.len() < in_bytes.len()); // no infinite loop            in_bytes = rest;            append_quoted_chunk(&mut out, cur_chunk, strategy);        }        Ok(out.into())    }}#[derive(PartialEq)]enum QuotingStrategy {    /// No quotes and no backslash escapes.  (If backslash escapes would be necessary, we use a    /// different strategy instead.)    Unquoted,    /// Single quoted.    SingleQuoted,    /// Double quotes, potentially with backslash escapes.    DoubleQuoted,    // TODO: add $'xxx' and "$(printf 'xxx')" styles}

/// Is this ASCII byte okay to emit unquoted?const fn unquoted_ok(c: u8) -> bool {    match c as char {        // Allowed characters:        '+' | '-' | '.' | '/' | ':' | '@' | ']' | '_' |        '0'..='9' | 'A'..='Z' | 'a'..='z'        => true,        // Non-allowed characters:        // From POSIX https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html        // "The application shall quote the following characters if they are to represent themselves:"        '|' | '&' | ';' | '<' | '>' | '(' | ')' | '$' | '`' | '\\' | '"' | '\'' | ' ' | '\t' | '\n' |        // "and the following may need to be quoted under certain circumstances[..]:"        '*' | '?' | '[' | '#' | '~' | '=' | '%' |        // Brace expansion.  These ought to be in the POSIX list but aren't yet;        // see: https://www.austingroupbugs.net/view.php?id=1193        '{' | '}' |        // Also quote comma, just to be safe in the extremely odd case that the user of this crate        // is intentionally placing a quoted string inside a brace expansion, e.g.:        //     format!("echo foo{{a,b,{}}}" | shlex::quote(some_str))        ',' |        // '\r' is allowed in a word by all real shells I tested, but is treated as a word        // separator by Python `shlex` | and might be translated to '\n' in interactive mode.        '\r' |        // '!' and '^' are treated specially in interactive mode; see quoting_warning.        '!' | '^' |        // Nul bytes and control characters.        '\x00' ..= '\x1f' | '\x7f'        => false,        '\u{80}' ..= '\u{10ffff}' => {            // This is unreachable since `unquoted_ok` is only called for 0..128.            // Non-ASCII bytes are handled separately in `quoting_strategy`.            // Can't call unreachable!() from `const fn` on old Rust, so...            unquoted_ok(c)        },    }    // Note: The logic cited above for quoting comma might suggest that `..` should also be quoted,    // it as a special case of brace expansion).  But it's not necessary.  There are three cases:    //    // 1. The user wants comma-based brace expansion, but the untrusted string being `quote`d    //    contains `..`, so they get something like `{foo,bar,3..5}`.    //  => That's safe; both Bash and Zsh expand this to `foo bar 3..5` rather than    //     `foo bar 3 4 5`.  The presence of commas disables sequence expression expansion.    //    // 2. The user wants comma-based brace expansion where the contents of the braces are a    //    variable number of `quote`d strings and nothing else.  There happens to be exactly    //    one string and it contains `..`, so they get something like `{3..5}`.    //  => Then this will expand as a sequence expression, which is unintended.  But I don't mind,    //     because any such code is already buggy.  Suppose the untrusted string *didn't* contain    //     `,` or `..`, resulting in shell input like `{foo}`.  Then the shell would interpret it    //     as the literal string `{foo}` rather than brace-expanding it into `foo`.    //    // 3. The user wants a sequence expression and wants to supply an untrusted string as one of    //    the endpoints or the increment.    //  => Well, that's just silly, since the endpoints can only be numbers or single letters.}/// Optimized version of `unquoted_ok`.fn unquoted_ok_fast(c: u8) -> bool {    const UNQUOTED_OK_MASK: u128 = {        // Make a mask of all bytes in 0..<0x80 that pass.        let mut c = 0u8;        let mut mask = 0u128;        while c < 0x80 {            if unquoted_ok(c) {                mask |= 1u128 << c;            }            c += 1;        }        mask    };    ((UNQUOTED_OK_MASK >> c) & 1) != 0}/// Is this ASCII byte okay to emit in single quotes?fn single_quoted_ok(c: u8) -> bool {    match c {        // No single quotes in single quotes.        b'\'' => false,        // To work around a Bash bug, ^ is only allowed right after an opening single quote; see        // quoting_warning.        b'^' => false,        // Backslashes in single quotes are literal according to POSIX, but Fish treats them as an        // escape character.  Ban them.  Fish doesn't aim to be POSIX-compatible, but we *can*        // achieve Fish compatibility using double quotes, so we might as well.        b'\\' => false,        _ => true    }}/// Is this ASCII byte okay to emit in double quotes?fn double_quoted_ok(c: u8) -> bool {    match c {        // Work around Python `shlex` bug where parsing "\`" and "\$" doesn't strip the        // backslash, even though POSIX requires it.        b'`' | b'$' => false,        // '!' and '^' are treated specially in interactive mode; see quoting_warning.        b'!' | b'^' => false,        _ => true    }}/// Given an input, return a quoting strategy that can cover some prefix of the string, along with/// the size of that prefix.////// Precondition: input size is nonzero.  (Empty strings are handled by the caller.)/// Postcondition: returned size is nonzero.#[cfg_attr(manual_codegen_check, inline(never))]fn quoting_strategy(in_bytes: &[u8]) -> (usize, QuotingStrategy) {    const UNQUOTED_OK: u8 = 1;    const SINGLE_QUOTED_OK: u8 = 2;    const DOUBLE_QUOTED_OK: u8 = 4;    let mut prev_ok = SINGLE_QUOTED_OK | DOUBLE_QUOTED_OK | UNQUOTED_OK;    let mut i = 0;    if in_bytes[0] == b'^' {        // To work around a Bash bug, ^ is only allowed right after an opening single quote; see        // quoting_warning.        prev_ok = SINGLE_QUOTED_OK;        i = 1;    }    while i < in_bytes.len() {        let c = in_bytes[i];        let mut cur_ok = prev_ok;        if c >= 0x80 {            // Normally, non-ASCII characters shouldn't require quoting, but see quoting_warning.md            // about \xa0.  For now, just treat all non-ASCII characters as requiring quotes.  This            // also ensures things are safe in the off-chance that you're in a legacy 8-bit locale that            // has additional characters satisfying `isblank`.            cur_ok &= !UNQUOTED_OK;        } else {            if !unquoted_ok_fast(c) {                cur_ok &= !UNQUOTED_OK;            }            if !single_quoted_ok(c){                cur_ok &= !SINGLE_QUOTED_OK;            }            if !double_quoted_ok(c) {                cur_ok &= !DOUBLE_QUOTED_OK;            }        }        if cur_ok == 0 {            // There are no quoting strategies that would work for both the previous characters and            // this one.  So we have to end the chunk before this character.  The caller will call            // `quoting_strategy` again to handle the rest of the string.            break;        }        prev_ok = cur_ok;        i += 1;    }    // Pick the best allowed strategy.    let strategy = if prev_ok & UNQUOTED_OK != 0 {        QuotingStrategy::Unquoted    } else if prev_ok & SINGLE_QUOTED_OK != 0 {        QuotingStrategy::SingleQuoted    } else if prev_ok & DOUBLE_QUOTED_OK != 0 {        QuotingStrategy::DoubleQuoted    } else {        unreachable!()    };    debug_assert!(i > 0);    (i, strategy)}fn append_quoted_chunk(out: &mut Vec<u8>, cur_chunk: &[u8], strategy: QuotingStrategy) {    match strategy {        QuotingStrategy::Unquoted => {            out.extend_from_slice(cur_chunk);        },        QuotingStrategy::SingleQuoted => {            out.reserve(cur_chunk.len() + 2);            out.push(b'\'');            out.extend_from_slice(cur_chunk);            out.push(b'\'');        },        QuotingStrategy::DoubleQuoted => {            out.reserve(cur_chunk.len() + 2);            out.push(b'"');            for &c in cur_chunk.iter() {                if let b'$' | b'`' | b'"' | b'\\' = c {                    // Add a preceding backslash.                    // Note: We shouldn't actually get here for $ and ` because they don't pass                    // `double_quoted_ok`.                    out.push(b'\\');                }                // Add the character itself.                out.push(c);            }            out.push(b'"');        },    }}

/// Convenience function that consumes an iterable of words and turns it into a single byte string,/// quoting words when necessary. Consecutive words will be separated by a single space.////// Uses default settings. The only error that can be returned is [`QuoteError::Nul`].////// Equivalent to [`Quoter::new().join(words)`](Quoter).////// The string equivalent is [shlex::try_join].pub fn try_join<'a, I: IntoIterator<Item = &'a [u8]>>(words: I) -> Result<Vec<u8>, QuoteError> {    Quoter::new().join(words)}/// Given a single word, return a string suitable to encode it as a shell argument.////// Uses default settings. The only error that can be returned is [`QuoteError::Nul`].////// Equivalent to [`Quoter::new().quote(in_bytes)`](Quoter).////// The string equivalent is [shlex::try_quote].pub fn try_quote(in_bytes: &[u8]) -> Result<Cow<'_, [u8]>, QuoteError> {    Quoter::new().quote(in_bytes)}#[cfg(test)]const INVALID_UTF8: &[u8] = b"\xa1";#[test]#[allow(invalid_from_utf8)]fn test_invalid_utf8() {    // Check that our test string is actually invalid UTF-8.    assert!(core::str::from_utf8(INVALID_UTF8).is_err());}#[cfg(test)]static SPLIT_TEST_ITEMS: &[(&[u8], Option<&[&[u8]]>)] = &[    (b"foo$baz", Some(&[b"foo$baz"])),    (b"foo baz", Some(&[b"foo", b"baz"])),    (b"foo\"bar\"baz", Some(&[b"foobarbaz"])),    (b"foo \"bar\"baz", Some(&[b"foo", b"barbaz"])),    (b"   foo \nbar", Some(&[b"foo", b"bar"])),    (b"foo\\\nbar", Some(&[b"foobar"])),    (b"\"foo\\\nbar\"", Some(&[b"foobar"])),    (b"'baz\\$b'", Some(&[b"baz\\$b"])),    (b"'baz\\\''", None),    (b"\\", None),    (b"\"\\", None),    (b"'\\", None),    (b"\"", None),    (b"'", None),    (b"foo #bar\nbaz", Some(&[b"foo", b"baz"])),    (b"foo #bar", Some(&[b"foo"])),    (b"foo#bar", Some(&[b"foo#bar"])),    (b"foo\"#bar", None),    (b"'\\n'", Some(&[b"\\n"])),    (b"'\\\\n'", Some(&[b"\\\\n"])),    (INVALID_UTF8, Some(&[INVALID_UTF8])),];#[test]fn test_split() {    for &(input, output) in SPLIT_TEST_ITEMS {        assert_eq!(split(input), output.map(|o| o.iter().map(|&x| x.to_owned()).collect()));    }}#[test]fn test_lineno() {    let mut sh = Shlex::new(b"\nfoo\nbar");    while let Some(word) = sh.next() {        if word == b"bar" {            assert_eq!(sh.line_no, 3);        }    }}