From 1b6a04ca5504955c571d1c97504fb45ea0befee4 Mon Sep 17 00:00:00 2001 From: Valentin Popov Date: Mon, 8 Jan 2024 01:21:28 +0400 Subject: Initial vendor packages Signed-off-by: Valentin Popov --- vendor/encode_unicode/src/decoding_iterators.rs | 494 +++++++++++ vendor/encode_unicode/src/errors.rs | 289 +++++++ vendor/encode_unicode/src/lib.rs | 78 ++ vendor/encode_unicode/src/traits.rs | 1014 +++++++++++++++++++++++ vendor/encode_unicode/src/utf16_char.rs | 687 +++++++++++++++ vendor/encode_unicode/src/utf16_iterators.rs | 270 ++++++ vendor/encode_unicode/src/utf8_char.rs | 647 +++++++++++++++ vendor/encode_unicode/src/utf8_iterators.rs | 352 ++++++++ 8 files changed, 3831 insertions(+) create mode 100644 vendor/encode_unicode/src/decoding_iterators.rs create mode 100644 vendor/encode_unicode/src/errors.rs create mode 100644 vendor/encode_unicode/src/lib.rs create mode 100644 vendor/encode_unicode/src/traits.rs create mode 100644 vendor/encode_unicode/src/utf16_char.rs create mode 100644 vendor/encode_unicode/src/utf16_iterators.rs create mode 100644 vendor/encode_unicode/src/utf8_char.rs create mode 100644 vendor/encode_unicode/src/utf8_iterators.rs (limited to 'vendor/encode_unicode/src') diff --git a/vendor/encode_unicode/src/decoding_iterators.rs b/vendor/encode_unicode/src/decoding_iterators.rs new file mode 100644 index 0000000..4ef4125 --- /dev/null +++ b/vendor/encode_unicode/src/decoding_iterators.rs @@ -0,0 +1,494 @@ +/* Copyright 2018 The encode_unicode Developers + * + * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be + * copied, modified, or distributed except according to those terms. + */ + +//! Iterators that turn multiple `u8`s or `u16`s into `Utf*Char`s, but can fail. +//! +//! To be predictable, all errors consume one element each. +//! +//! The iterator adaptors produce neither offset nor element length to work +//! well with other adaptors, +//! while the slice iterators yield both to make more advanced use cases easy. + +use errors::{InvalidUtf8Slice, InvalidUtf16FirstUnit, Utf16PairError}; +use errors::InvalidUtf8Slice::*; +use errors::InvalidUtf8::*; +use errors::InvalidUtf8FirstByte::*; +use errors::InvalidUtf16Slice::*; +use errors::InvalidCodepoint::*; +use errors::Utf16PairError::*; +use utf8_char::Utf8Char; +use utf16_char::Utf16Char; +use traits::U16UtfExt; +extern crate core; +use self::core::borrow::Borrow; +use self::core::fmt::{self, Debug}; +use self::core::iter::Chain; +use self::core::option; + + +/// Decodes UTF-8 characters from a byte iterator into `Utf8Char`s. +/// +/// See [`IterExt::to_utf8chars()`](../trait.IterExt.html#tymethod.to_utf8chars) +/// for examples and error handling. +#[derive(Clone, Default)] +pub struct Utf8CharMerger, I:Iterator> { + iter: I, + /// number of bytes that were read before an error was detected + after_err_leftover: u8, + /// stack because it simplifies popping. + after_err_stack: [u8; 3], +} +impl, I:Iterator, T:IntoIterator> +From for Utf8CharMerger { + fn from(t: T) -> Self { + Utf8CharMerger { + iter: t.into_iter(), + after_err_leftover: 0, + after_err_stack: [0; 3], + } + } +} +impl, I:Iterator> Utf8CharMerger { + /// Extract the inner iterator. + /// + /// If the last item produced by `.next()` was an `Err`, + /// up to three following bytes might be missing. + /// The exact number of missing bytes for each error type should not be relied on. + /// + /// # Examples + /// + /// Three bytes swallowed: + /// ``` + /// # use encode_unicode::IterExt; + /// let mut merger = b"\xf4\xa1\xb2FS".iter().to_utf8chars(); + /// assert!(merger.next().unwrap().is_err()); + /// let mut inner: std::slice::Iter = merger.into_inner(); + /// assert_eq!(inner.next(), Some(&b'S')); // b'\xa1', b'\xb2' and b'F' disappeared + /// ``` + /// + /// All bytes present: + /// ``` + /// # use encode_unicode::IterExt; + /// let mut merger = b"\xb0FS".iter().to_utf8chars(); + /// assert!(merger.next().unwrap().is_err()); + /// assert_eq!(merger.into_inner().next(), Some(&b'F')); + /// ``` + /// + /// Two bytes missing: + /// ``` + /// # use encode_unicode::IterExt; + /// let mut merger = b"\xe0\x80\x80FS".iter().to_utf8chars(); + /// assert!(merger.next().unwrap().is_err()); + /// assert_eq!(merger.into_inner().next(), Some(&b'F')); + /// ``` + pub fn into_inner(self) -> I { + self.iter + } + + fn save(&mut self, bytes: &[u8;4], len: usize) { + // forget bytes[0] and push the others onto self.after_err_stack (in reverse). + for &after_err in bytes[1..len].iter().rev() { + self.after_err_stack[self.after_err_leftover as usize] = after_err; + self.after_err_leftover += 1; + } + } + /// Reads len-1 bytes into bytes[1..] + fn extra(&mut self, bytes: &mut[u8;4], len: usize) -> Result<(),InvalidUtf8Slice> { + // This is the only function that pushes onto after_err_stack, + // and it checks that all bytes are continuation bytes before fetching the next one. + // Therefore only the last byte retrieved can be a non-continuation byte. + // That last byte is also the last to be retrieved from after_err. + // + // Before this function is called, there has been retrieved at least one byte. + // If that byte was a continuation byte, next() produces an error + // and won't call this function. + // Therefore, we know that after_err is empty at this point. + // This means that we can use self.iter directly, and knows where to start pushing + debug_assert_eq!(self.after_err_leftover, 0, "first: {:#02x}, stack: {:?}", bytes[0], self.after_err_stack); + for i in 1..len { + if let Some(extra) = self.iter.next() { + let extra = *extra.borrow(); + bytes[i] = extra; + if extra & 0b1100_0000 != 0b1000_0000 { + // not a continuation byte + self.save(bytes, i+1); + return Err(InvalidUtf8Slice::Utf8(NotAContinuationByte(i))) + } + } else { + self.save(bytes, i); + return Err(TooShort(len)); + } + } + Ok(()) + } +} +impl, I:Iterator> Iterator for Utf8CharMerger { + type Item = Result; + fn next(&mut self) -> Option { + let first: u8; + if self.after_err_leftover != 0 { + self.after_err_leftover -= 1; + first = self.after_err_stack[self.after_err_leftover as usize]; + } else if let Some(next) = self.iter.next() { + first = *next.borrow(); + } else { + return None; + } + + unsafe { + let mut bytes = [first, 0, 0, 0]; + let ok = match first { + 0b0000_0000...0b0111_1111 => {/*1 and */Ok(())}, + 0b1100_0010...0b1101_1111 => {//2 and not overlong + self.extra(&mut bytes, 2) // no extra validation required + }, + 0b1110_0000...0b1110_1111 => {//3 + if let Err(e) = self.extra(&mut bytes, 3) { + Err(e) + } else if bytes[0] == 0b1110_0000 && bytes[1] <= 0b10_011111 { + self.save(&bytes, 3); + Err(Utf8(OverLong)) + } else if bytes[0] == 0b1110_1101 && bytes[1] & 0b11_100000 == 0b10_100000 { + self.save(&bytes, 3); + Err(Codepoint(Utf16Reserved)) + } else { + Ok(()) + } + }, + 0b1111_0000...0b1111_0100 => {//4 + if let Err(e) = self.extra(&mut bytes, 4) { + Err(e) + } else if bytes[0] == 0b11110_000 && bytes[1] <= 0b10_001111 { + self.save(&bytes, 4); + Err(InvalidUtf8Slice::Utf8(OverLong)) + } else if bytes[0] == 0b11110_100 && bytes[1] > 0b10_001111 { + self.save(&bytes, 4); + Err(InvalidUtf8Slice::Codepoint(TooHigh)) + } else { + Ok(()) + } + }, + 0b1000_0000...0b1011_1111 => {// continuation byte + Err(Utf8(FirstByte(ContinuationByte))) + }, + 0b1100_0000...0b1100_0001 => {// 2 and overlong + Err(Utf8(OverLong)) + }, + 0b1111_0101...0b1111_0111 => {// 4 and too high codepoint + Err(Codepoint(TooHigh)) + }, + 0b1111_1000...0b1111_1111 => { + Err(Utf8(FirstByte(TooLongSeqence))) + }, + _ => unreachable!("all possible byte values should be covered") + }; + Some(ok.map(|()| Utf8Char::from_array_unchecked(bytes) )) + } + } + fn size_hint(&self) -> (usize,Option) { + let (iter_min, iter_max) = self.iter.size_hint(); + // cannot be exact, so KISS + let min = iter_min / 4; // don't bother rounding up or accounting for after_err + // handle edge case of max > usize::MAX-3 just in case. + // Using wrapping_add() wouldn't violate any API contract as the trait isn't unsafe. + let max = iter_max.and_then(|max| { + max.checked_add(self.after_err_leftover as usize) + }); + (min, max) + } +} +impl, I:Iterator+Debug> Debug for Utf8CharMerger { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + let mut in_order = [0u8; 3]; + for i in 0..self.after_err_leftover as usize { + in_order[i] = self.after_err_stack[self.after_err_leftover as usize - i - 1]; + } + fmtr.debug_struct("Utf8CharMerger") + .field("buffered", &&in_order[..self.after_err_leftover as usize]) + .field("inner", &self.iter) + .finish() + } +} + + +/// An [`Utf8CharMerger`](struct.Utf8CharMerger.html) that also produces +/// offsets and lengths, but can only iterate over slices. +/// +/// See [`SliceExt::utf8char_indices()`](../trait.SliceExt.html#tymethod.utf8char_indices) +/// for examples and error handling. +#[derive(Clone, Default)] +pub struct Utf8CharDecoder<'a> { + slice: &'a[u8], + index: usize, +} +impl<'a> From<&'a[u8]> for Utf8CharDecoder<'a> { + fn from(s: &[u8]) -> Utf8CharDecoder { + Utf8CharDecoder { slice: s, index: 0 } + } +} +impl<'a> Utf8CharDecoder<'a> { + /// Extract the remainder of the source slice. + /// + /// # Examples + /// + /// Unlike `Utf8CharMerger::into_inner()`, bytes directly after an error + /// are never swallowed: + /// ``` + /// # use encode_unicode::SliceExt; + /// let mut iter = b"\xf4\xa1\xb2FS".utf8char_indices(); + /// assert!(iter.next().unwrap().1.is_err()); + /// assert_eq!(iter.as_slice(), b"\xa1\xb2FS"); + /// ``` + pub fn as_slice(&self) -> &'a[u8] { + &self.slice[self.index..] + } +} +impl<'a> Iterator for Utf8CharDecoder<'a> { + type Item = (usize, Result, usize); + fn next(&mut self) -> Option { + let start = self.index; + match Utf8Char::from_slice_start(&self.slice[self.index..]) { + Ok((u8c, len)) => { + self.index += len; + Some((start, Ok(u8c), len)) + }, + Err(TooShort(1)) => None, + Err(e) => { + self.index += 1; + Some((start, Err(e), 1)) + } + } + } + #[inline] + fn size_hint(&self) -> (usize,Option) { + let bytes = self.slice.len() - self.index; + // Cannot be exact, so KISS and don't bother rounding up. + // The slice is unlikely be full of 4-byte codepoints, so buffers + // allocated with the lower bound will have to be grown anyway. + (bytes/4, Some(bytes)) + } +} +impl<'a> DoubleEndedIterator for Utf8CharDecoder<'a> { + fn next_back(&mut self) -> Option { + if self.index < self.slice.len() { + let extras = self.slice.iter() + .rev() + .take_while(|&b| b & 0b1100_0000 == 0b1000_0000 ) + .count(); + let starts = self.slice.len() - (extras+1); + match Utf8Char::from_slice_start(&self.slice[starts..]) { + Ok((u8c,len)) if len == 1+extras => { + self.slice = &self.slice[..starts]; + Some((starts, Ok(u8c), len)) + }, + // This enures errors for every byte in both directions, + // but means overlong and codepoint errors will be turned into + // tooshort errors. + Err(e) if extras == 0 => { + self.slice = &self.slice[..self.slice.len()-1]; + Some((self.slice.len()-1, Err(e), 1)) + }, + _ => { + self.slice = &self.slice[..self.slice.len()-1]; + Some((self.slice.len()-1, Err(Utf8(FirstByte(ContinuationByte))), 1)) + }, + } + } else { + None + } + } +} +impl<'a> Debug for Utf8CharDecoder<'a> { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + write!(fmtr, "Utf8CharDecoder {{ bytes[{}..]: {:?} }}", self.index, self.as_slice()) + } +} + + + +/// Decodes UTF-16 characters from a `u16` iterator into `Utf16Char`s. +/// +/// See [`IterExt::to_utf16chars()`](../trait.IterExt.html#tymethod.to_utf16chars) +/// for examples and error handling. +#[derive(Clone, Default)] +pub struct Utf16CharMerger, I:Iterator> { + iter: I, + /// Used when a trailing surrogate was expected, the u16 can be any value. + prev: Option, +} +impl, I:Iterator, T:IntoIterator> +From for Utf16CharMerger { + fn from(t: T) -> Self { + Utf16CharMerger { iter: t.into_iter(), prev: None } + } +} +impl, I:Iterator> Utf16CharMerger { + /// Extract the inner iterator. + /// + /// If the last item produced was an `Err`, the first unit might be missing. + /// + /// # Examples + /// + /// Unit right after an error missing + /// ``` + /// # use encode_unicode::IterExt; + /// # use encode_unicode::error::Utf16PairError; + /// let mut merger = [0xd901, 'F' as u16, 'S' as u16].iter().to_utf16chars(); + /// assert_eq!(merger.next(), Some(Err(Utf16PairError::UnmatchedLeadingSurrogate))); + /// let mut inner: std::slice::Iter = merger.into_inner(); + /// assert_eq!(inner.next(), Some('S' as u16).as_ref()); // 'F' was consumed by Utf16CharMerger + /// ``` + /// + /// Error that doesn't swallow any units + /// ``` + /// # use encode_unicode::IterExt; + /// # use encode_unicode::error::Utf16PairError; + /// let mut merger = [0xde00, 'F' as u16, 'S' as u16].iter().to_utf16chars(); + /// assert_eq!(merger.next(), Some(Err(Utf16PairError::UnexpectedTrailingSurrogate))); + /// let mut inner: std::slice::Iter = merger.into_inner(); + /// assert_eq!(inner.next(), Some('F' as u16).as_ref()); // not consumed + /// ``` + pub fn into_inner(self) -> I { + self.iter + } + /// Returns an iterator over the remaining units. + /// Unlike `into_inner()` this will never drop any units. + /// + /// The exact type of the returned iterator should not be depended on. + /// + /// # Examples + /// + /// ``` + /// # use encode_unicode::IterExt; + /// # use encode_unicode::error::Utf16PairError; + /// let slice = [0xd901, 'F' as u16, 'S' as u16]; + /// let mut merger = slice.iter().to_utf16chars(); + /// assert_eq!(merger.next(), Some(Err(Utf16PairError::UnmatchedLeadingSurrogate))); + /// let mut remaining = merger.into_remaining_units(); + /// assert_eq!(remaining.next(), Some('F' as u16).as_ref()); + /// ``` + pub fn into_remaining_units(self) -> Chain,I> { + self.prev.into_iter().chain(self.iter) + } +} +impl, I:Iterator> Iterator for Utf16CharMerger { + type Item = Result; + fn next(&mut self) -> Option { + let first = self.prev.take().or_else(|| self.iter.next() ); + first.map(|first| unsafe { + match first.borrow().utf16_needs_extra_unit() { + Ok(false) => Ok(Utf16Char::from_array_unchecked([*first.borrow(), 0])), + Ok(true) => match self.iter.next() { + Some(second) => match second.borrow().utf16_needs_extra_unit() { + Err(InvalidUtf16FirstUnit) => Ok(Utf16Char::from_tuple_unchecked(( + *first.borrow(), + Some(*second.borrow()) + ))), + Ok(_) => { + self.prev = Some(second); + Err(Utf16PairError::UnmatchedLeadingSurrogate) + } + }, + None => Err(Utf16PairError::Incomplete) + }, + Err(InvalidUtf16FirstUnit) => Err(Utf16PairError::UnexpectedTrailingSurrogate), + } + }) + } + fn size_hint(&self) -> (usize,Option) { + let (iter_min, iter_max) = self.iter.size_hint(); + // cannot be exact, so KISS + let min = iter_min / 2; // don't bother rounding up or accounting for self.prev + let max = match (iter_max, &self.prev) { + (Some(max), &Some(_)) => max.checked_add(1), + (max, _) => max, + }; + (min, max) + } +} +impl, I:Iterator+Debug> Debug for Utf16CharMerger { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + fmtr.debug_struct("Utf16CharMerger") + .field("buffered", &self.prev.as_ref().map(|b| *b.borrow() )) + .field("inner", &self.iter) + .finish() + } +} + + +/// An [`Utf16CharMerger`](struct.Utf16CharMerger.html) that also produces +/// offsets and lengths, but can only iterate over slices. +/// +/// See [`SliceExt::utf16char_indices()`](../trait.SliceExt.html#tymethod.utf16char_indices) +/// for examples and error handling. +#[derive(Clone, Default)] +pub struct Utf16CharDecoder<'a> { + slice: &'a[u16], + index: usize, +} +impl<'a> From<&'a[u16]> for Utf16CharDecoder<'a> { + fn from(s: &'a[u16]) -> Self { + Utf16CharDecoder{ slice: s, index: 0 } + } +} +impl<'a> Utf16CharDecoder<'a> { + /// Extract the remainder of the source slice. + /// + /// # Examples + /// + /// Unlike `Utf16CharMerger::into_inner()`, the unit after an error is never swallowed: + /// ``` + /// # use encode_unicode::SliceExt; + /// # use encode_unicode::error::Utf16PairError; + /// let mut iter = [0xd901, 'F' as u16, 'S' as u16].utf16char_indices(); + /// assert_eq!(iter.next(), Some((0, Err(Utf16PairError::UnmatchedLeadingSurrogate), 1))); + /// assert_eq!(iter.as_slice(), &['F' as u16, 'S' as u16]); + /// ``` + pub fn as_slice(&self) -> &[u16] { + &self.slice[self.index..] + } +} +impl<'a> Iterator for Utf16CharDecoder<'a> { + type Item = (usize,Result,usize); + #[inline] + fn next(&mut self) -> Option { + let start = self.index; + match Utf16Char::from_slice_start(self.as_slice()) { + Ok((u16c,len)) => { + self.index += len; + Some((start, Ok(u16c), len)) + }, + Err(EmptySlice) => None, + Err(FirstLowSurrogate) => { + self.index += 1; + Some((start, Err(UnexpectedTrailingSurrogate), 1)) + }, + Err(SecondNotLowSurrogate) => { + self.index += 1; + Some((start, Err(UnmatchedLeadingSurrogate), 1)) + }, + Err(MissingSecond) => { + self.index = self.slice.len(); + Some((start, Err(Incomplete), 1)) + } + } + } + #[inline] + fn size_hint(&self) -> (usize,Option) { + let units = self.slice.len() - self.index; + // Cannot be exact, so KISS and don't bother rounding up. + // The slice is unlikely be full of surrogate pairs, so buffers + // allocated with the lower bound will have to be grown anyway. + (units/2, Some(units)) + } +} +impl<'a> Debug for Utf16CharDecoder<'a> { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + write!(fmtr, "Utf16CharDecoder {{ units[{}..]: {:?} }}", self.index, self.as_slice()) + } +} diff --git a/vendor/encode_unicode/src/errors.rs b/vendor/encode_unicode/src/errors.rs new file mode 100644 index 0000000..fb587fa --- /dev/null +++ b/vendor/encode_unicode/src/errors.rs @@ -0,0 +1,289 @@ +/* Copyright 2016 The encode_unicode Developers + * + * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be + * copied, modified, or distributed except according to those terms. + */ + + +//! Boilerplatey error types + +extern crate core; +use self::core::fmt::{self,Display,Formatter}; +#[cfg(feature="std")] +use std::error::Error; + + +macro_rules! description {($err:ty, $desc:expr) => { + #[cfg(not(feature="std"))] + impl $err { + #[allow(missing_docs)] + pub fn description(&self) -> &'static str { + ($desc)(self) + } + } + #[cfg(feature="std")] + impl Error for $err { + fn description(&self) -> &'static str { + ($desc)(self) + } + } + impl Display for $err { + fn fmt(&self, fmtr: &mut Formatter) -> fmt::Result { + write!(fmtr, "{}", self.description()) + } + } +}} + + +macro_rules! single_cause {(#[$doc1:meta] #[$doc2:meta] $err:ident => $desc:expr) => { + // Rust 1.15 doesn't understand $(#[$doc:meta])* $:ident + #[$doc1] + #[$doc2] + #[derive(Clone,Copy, Debug, PartialEq,Eq)] + pub struct $err; + description!{$err, |_| $desc } +}} + + +single_cause!{ + /// Cannot tell whether an `u16` needs an extra unit, + /// because it's a trailing surrogate itself. + InvalidUtf16FirstUnit => "is a trailing surrogate" +} + +single_cause!{ + /// Cannot create an `Utf8Char` or `Utf16Char` from the first codepoint of a str, + /// because there are none. + EmptyStrError => "is empty" +} + +single_cause!{ + /// Cannot create an `Utf8Char` from a standalone `u8` + /// that is not an ASCII character. + NonAsciiError => "is not an ASCII character" +} + +single_cause!{ + /// Cannot create an `Utf16Char` from a standalone `u16` that is not a + /// codepoint in the basic multilingual plane, but part of a suurrogate pair. + NonBMPError => "is not a codepoint in the basic multilingual plane" +} + + + +macro_rules! simple {(#[$tydoc:meta] $err:ident { + $($(#[$vardoc:meta])* ::$variant:ident => $string:expr),+, + } ) => { + #[$tydoc] + #[derive(Clone,Copy, Debug, PartialEq,Eq)] + pub enum $err { + $($(#[$vardoc])* $variant),* + } + description!{$err, |e: &$err| match *e {$($err::$variant=>$string),*} } +}} + + +simple!{/// Reasons why an `u32` is not a valid UTF codepoint. + InvalidCodepoint { + /// It's reserved for UTF-16 surrogate pairs." + ::Utf16Reserved => "is reserved for UTF-16 surrogate pairs", + /// It's higher than the highest codepoint (which is 0x10ffff). + ::TooHigh => "is higher than the highest codepoint", + }} +use self::InvalidCodepoint::*; +impl InvalidCodepoint { + /// Get the range of values for which this error would be given. + pub fn error_range(self) -> (u32,u32) {match self { + Utf16Reserved => (0xd8_00, 0xdf_ff), + TooHigh => (0x00_10_ff_ff, 0xff_ff_ff_ff), + }} +} + + +simple!{/// Reasons why a `[u16; 2]` doesn't form a valid UTF-16 codepoint. + InvalidUtf16Array { + /// The first unit is a trailing/low surrogate, which is never valid. + ::FirstIsTrailingSurrogate => "the first unit is a trailing surrogate, which is never valid", + /// The second unit is needed, but is not a trailing surrogate. + ::SecondIsNotTrailingSurrogate => "the second unit is needed but is not a trailing surrogate", + }} + +simple!{/// Reasons why one or two `u16`s are not valid UTF-16, in sinking precedence. + InvalidUtf16Tuple { + /// The first unit is a trailing/low surrogate, which is never valid. + /// + /// Note that the value of a low surrogate is actually higher than a high surrogate. + ::FirstIsTrailingSurrogate => "the first unit is a trailing / low surrogate, which is never valid", + /// You provided a second unit, but the first one stands on its own. + ::SuperfluousSecond => "the second unit is superfluous", + /// The first and only unit requires a second unit. + ::MissingSecond => "the first unit requires a second unit", + /// The first unit requires a second unit, but it's not a trailing/low surrogate. + /// + /// Note that the value of a low surrogate is actually higher than a high surrogate. + ::InvalidSecond => "the required second unit is not a trailing / low surrogate", + }} + + +simple!{/// Reasons why a slice of `u16`s doesn't start with valid UTF-16. + InvalidUtf16Slice { + /// The slice is empty. + ::EmptySlice => "the slice is empty", + /// The first unit is a low surrogate. + ::FirstLowSurrogate => "the first unit is a trailing surrogate", + /// The first and only unit requires a second unit. + ::MissingSecond => "the first and only unit requires a second one", + /// The first unit requires a second one, but it's not a trailing surrogate. + ::SecondNotLowSurrogate => "the required second unit is not a trailing surrogate", + }} + +simple!{/// Types of invalid sequences encountered by `Utf16CharParser`. + Utf16PairError { + /// A trailing surrogate was not preceeded by a leading surrogate. + ::UnexpectedTrailingSurrogate => "a trailing surrogate was not preceeded by a leading surrogate", + /// A leading surrogate was followed by an unit that was not a trailing surrogate. + ::UnmatchedLeadingSurrogate => "a leading surrogate was followed by an unit that was not a trailing surrogate", + /// A trailing surrogate was expected when the end was reached. + ::Incomplete => "a trailing surrogate was expected when the end was reached", + }} + + +simple!{/// Reasons why `Utf8Char::from_str()` or `Utf16Char::from_str()` failed. + FromStrError { + /// `Utf8Char` or `Utf16Char` cannot store more than a single codepoint. + ::MultipleCodepoints => "has more than one codepoint", + /// `Utf8Char` or `Utf16Char` cannot be empty. + ::Empty => "is empty", + }} + + +simple!{/// Reasons why a byte is not the start of a UTF-8 codepoint. + InvalidUtf8FirstByte { + /// Sequences cannot be longer than 4 bytes. Is given for values >= 240. + ::TooLongSeqence => "is greater than 247 (UTF-8 sequences cannot be longer than four bytes)", + /// This byte belongs to a previous sequence. Is given for values between 128 and 192 (exclusive). + ::ContinuationByte => "is a continuation of a previous sequence", + }} +use self::InvalidUtf8FirstByte::*; + + + +macro_rules! complex { +($err:ty + {$($sub:ty => $to:expr,)*} + {$($desc:pat => $string:expr),+,} + => $use_cause:expr => + {$($cause:pat => $result:expr),+,} $(#[$causedoc:meta])* +) => { + $(impl From<$sub> for $err { + fn from(error: $sub) -> $err { + $to(error) + } + })* + #[cfg(not(feature="std"))] + impl $err { + #[allow(missing_docs)] + pub fn description(&self) -> &'static str { + match *self{ $($desc => $string,)* } + } + /// A hack to avoid two Display impls + fn cause(&self) -> Option<&Display> {None} + } + #[cfg(feature="std")] + impl Error for $err { + fn description(&self) -> &'static str { + match *self{ $($desc => $string,)* } + } + $(#[$causedoc])* + fn cause(&self) -> Option<&Error> { + match *self{ $($cause => $result,)* } + } + } + impl Display for $err { + fn fmt(&self, fmtr: &mut Formatter) -> fmt::Result { + match (self.cause(), $use_cause) { + (Some(d),true) => write!(fmtr, "{}: {}", self.description(), d), + _ => write!(fmtr, "{}", self.description()), + } + } + } +}} + + +/// Reasons why a byte sequence is not valid UTF-8, excluding invalid codepoint. +/// In sinking precedence. +#[derive(Clone,Copy, Debug, PartialEq,Eq)] +pub enum InvalidUtf8 { + /// Something is wrong with the first byte. + FirstByte(InvalidUtf8FirstByte), + /// The byte at index 1...3 should be a continuation byte, + /// but dosesn't fit the pattern 0b10xx_xxxx. + NotAContinuationByte(usize), + /// There are too many leading zeros: it could be a byte shorter. + /// + /// [Decoding this could allow someone to input otherwise prohibited + /// characters and sequences, such as "../"](https://tools.ietf.org/html/rfc3629#section-10). + OverLong, +} +use self::InvalidUtf8::*; +complex!{InvalidUtf8 { + InvalidUtf8FirstByte => FirstByte, + } { + FirstByte(TooLongSeqence) => "the first byte is greater than 239 (UTF-8 sequences cannot be longer than four bytes)", + FirstByte(ContinuationByte) => "the first byte is a continuation of a previous sequence", + OverLong => "the sequence contains too many zeros and could be shorter", + NotAContinuationByte(_) => "the sequence is too short", + } => false => { + FirstByte(ref cause) => Some(cause), + _ => None, + }/// Returns `Some` if the error is a `InvalidUtf8FirstByte`. +} + + +/// Reasons why a byte array is not valid UTF-8, in sinking precedence. +#[derive(Clone,Copy, Debug, PartialEq,Eq)] +pub enum InvalidUtf8Array { + /// Not a valid UTF-8 sequence. + Utf8(InvalidUtf8), + /// Not a valid unicode codepoint. + Codepoint(InvalidCodepoint), +} +complex!{InvalidUtf8Array { + InvalidUtf8 => InvalidUtf8Array::Utf8, + InvalidCodepoint => InvalidUtf8Array::Codepoint, + } { + InvalidUtf8Array::Utf8(_) => "the sequence is invalid UTF-8", + InvalidUtf8Array::Codepoint(_) => "the encoded codepoint is invalid", + } => true => { + InvalidUtf8Array::Utf8(ref u) => Some(u), + InvalidUtf8Array::Codepoint(ref c) => Some(c), + }/// Always returns `Some`. +} + + +/// Reasons why a byte slice is not valid UTF-8, in sinking precedence. +#[derive(Clone,Copy, Debug, PartialEq,Eq)] +pub enum InvalidUtf8Slice { + /// Something is certainly wrong with the first byte. + Utf8(InvalidUtf8), + /// The encoded codepoint is invalid: + Codepoint(InvalidCodepoint), + /// The slice is too short; n bytes was required. + TooShort(usize), +} +complex!{InvalidUtf8Slice { + InvalidUtf8 => InvalidUtf8Slice::Utf8, + InvalidCodepoint => InvalidUtf8Slice::Codepoint, + } { + InvalidUtf8Slice::Utf8(_) => "the sequence is invalid UTF-8", + InvalidUtf8Slice::Codepoint(_) => "the encoded codepoint is invalid", + InvalidUtf8Slice::TooShort(1) => "the slice is empty", + InvalidUtf8Slice::TooShort(_) => "the slice is shorter than the sequence", + } => true => { + InvalidUtf8Slice::Utf8(ref u) => Some(u), + InvalidUtf8Slice::Codepoint(ref c) => Some(c), + InvalidUtf8Slice::TooShort(_) => None, + } +} diff --git a/vendor/encode_unicode/src/lib.rs b/vendor/encode_unicode/src/lib.rs new file mode 100644 index 0000000..c50e126 --- /dev/null +++ b/vendor/encode_unicode/src/lib.rs @@ -0,0 +1,78 @@ +/* Copyright 2016 The encode_unicode Developers + * + * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be + * copied, modified, or distributed except according to those terms. + */ + + +/*! +Miscellaneous UTF-8 and UTF-16 types and methods. + +# Optional features: +* `#![no_std]`-mode: There are a few differences: + * `Error` doesn't exist, but `description()` is made available as an inherent impl. + * `Extend`/`FromIterator`-implementations for `String`/`Vec`/`Vec` are missing. + * There is no `io`, so `Utf8Iterator` and `Utf8CharSplitter` doesn't implement `Read`. + + This feature is enabled by setting `default-features=false` in `Cargo.toml`: + `encode_unicode = {version="0.3.4", default-features=false}` +* Integration with the [ascii](https://tomprogrammer.github.io/rust-ascii/ascii/index.html) crate: + Convert `Utf8Char` and `Utf16Char` to and from + [ascii::`AsciiChar`](https://tomprogrammer.github.io/rust-ascii/ascii/enum.AsciiChar.html). + +The minimum supported version of Rust is 1.15, +older versions might work now but can break with a minor update. + +[crates.io page](https://crates.io/crates/encode_unicode) +[github repository](https://github.com/tormol/encode_unicode) + +*/ + +#![warn(missing_docs)] + +#![cfg_attr(not(feature="std"), no_std)] +// either `cargo clippy` doesn't see theese, or I get a warning when I build. +#![cfg_attr(feature="clippy", feature(plugin))] +#![cfg_attr(feature="clippy", plugin(clippy))] +#![cfg_attr(feature="clippy", allow(derive_hash_xor_eq))]// tested +#![cfg_attr(feature="clippy", allow(len_without_is_empty))]// UtfxChar is never empty +#![cfg_attr(feature="clippy", allow(match_same_arms))]// looks better IMO +#![cfg_attr(feature="clippy", allow(needless_return))]// `foo.bar(); foo` looks unfinished +#![cfg_attr(feature="clippy", allow(redundant_closure))]// keep it explicit +#![cfg_attr(feature="clippy", allow(redundant_closure_call))]// not redundant in macros +#![cfg_attr(feature="clippy", allow(cast_lossless))]// too much noise (and too verbose) +// precedence: I prefer spaces to parentheses, but it's nice to recheck. + +mod errors; +mod traits; +mod utf8_char; +mod utf8_iterators; +mod utf16_char; +mod utf16_iterators; +mod decoding_iterators; + +pub use traits::{CharExt, U8UtfExt, U16UtfExt, StrExt, IterExt, SliceExt}; +pub use utf8_char::Utf8Char; +pub use utf16_char::Utf16Char; +pub use utf8_iterators::{Utf8Iterator, iter_bytes}; +pub use utf16_iterators::{Utf16Iterator, iter_units}; + +pub mod error {// keeping the public interface in one file + //! Errors returned by various conversion methods in this crate. + pub use errors::{FromStrError, EmptyStrError}; + pub use errors::{InvalidCodepoint, InvalidUtf8}; + pub use errors::{InvalidUtf8FirstByte,InvalidUtf16FirstUnit}; + pub use errors::{InvalidUtf8Slice,InvalidUtf16Slice}; + pub use errors::{InvalidUtf8Array,InvalidUtf16Array,InvalidUtf16Tuple}; + pub use errors::Utf16PairError; +} + +pub mod iterator { + //! Iterator types that you should rarely need to name + pub use utf8_iterators::{Utf8Iterator, Utf8CharSplitter, Utf8Chars, Utf8CharIndices}; + pub use utf16_iterators::{Utf16Iterator, Utf16CharSplitter, Utf16Chars, Utf16CharIndices}; + pub use decoding_iterators::{Utf8CharMerger, Utf8CharDecoder}; + pub use decoding_iterators::{Utf16CharMerger, Utf16CharDecoder}; +} diff --git a/vendor/encode_unicode/src/traits.rs b/vendor/encode_unicode/src/traits.rs new file mode 100644 index 0000000..3f36903 --- /dev/null +++ b/vendor/encode_unicode/src/traits.rs @@ -0,0 +1,1014 @@ +/* Copyright 2016 The encode_unicode Developers + * + * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be + * copied, modified, or distributed except according to those terms. + */ + +#![allow(unused_unsafe)]// explicit unsafe{} blocks in unsafe functions are a good thing. + +use utf8_char::Utf8Char; +use utf16_char::Utf16Char; +use utf8_iterators::*; +use utf16_iterators::*; +use decoding_iterators::*; +use error::*; +extern crate core; +use self::core::{char, u32, mem}; +use self::core::ops::{Not, Index, RangeFull}; +use self::core::borrow::Borrow; +#[cfg(feature="ascii")] +extern crate ascii; +#[cfg(feature="ascii")] +use self::ascii::AsciiStr; + +// TODO better docs and tests + +/// Methods for working with `u8`s as UTF-8 bytes. +pub trait U8UtfExt { + /// How many more bytes will you need to complete this codepoint? + /// + /// # Errors + /// + /// An error is returned if the byte is not a valid start of an UTF-8 + /// codepoint: + /// + /// * `128..192`: ContinuationByte + /// * `248..`: TooLongSequence + /// + /// Values in 244..248 represent a too high codepoint, but do not cause an + /// error. + fn extra_utf8_bytes(self) -> Result; + + /// How many more bytes will you need to complete this codepoint? + /// + /// This function assumes that the byte is a valid UTF-8 start, and might + /// return any value otherwise. (but the function is pure and safe to call + /// with any value). + fn extra_utf8_bytes_unchecked(self) -> usize; +} + +impl U8UtfExt for u8 { + #[inline] + fn extra_utf8_bytes(self) -> Result { + use error::InvalidUtf8FirstByte::{ContinuationByte,TooLongSeqence}; + // the bit twiddling is explained in extra_utf8_bytes_unchecked() + if self < 128 { + return Ok(0); + } + match ((self as u32)<<25).not().leading_zeros() { + n @ 1...3 => Ok(n as usize), + 0 => Err(ContinuationByte), + _ => Err(TooLongSeqence), + } + } + #[inline] + fn extra_utf8_bytes_unchecked(self) -> usize { + // For fun I've optimized this function (for x86 instruction count): + // The most straightforward implementation, that lets the compiler do + // the optimizing: + //match self { + // 0b0000_0000...0b0111_1111 => 0, + // 0b1100_0010...0b1101_1111 => 1, + // 0b1110_0000...0b1110_1111 => 2, + // 0b1111_0000...0b1111_0100 => 3, + // _ => whatever() + //} + // Using `unsafe{self::core::hint::unreachable_unchecked()}` for the + // "don't care" case is a terrible idea: while having the function + // non-deterministically return whatever happens to be in a register + // MIGHT be acceptable, it permits the function to not `ret`urn at all, + // but let execution fall through to whatever comes after it in the + // binary! (in other words completely UB). + // Currently unreachable_unchecked() might trap too, + // which is certainly not what we want. + // I also think `unsafe{mem::unitialized()}` is much more likely to + // explicitly produce whatever happens to be in a register than tell + // the compiler it can ignore this branch but needs to produce a value. + // + // From the bit patterns we see that for non-ASCII values the result is + // (number of leading set bits) - 1 + // The standard library doesn't have a method for counting leading ones, + // but it has leading_zeros(), which can be used after inverting. + // This function can therefore be reduced to the one-liner + //`self.not().leading_zeros().saturating_sub(1) as usize`, which would + // be branchless for architectures with instructions for + // leading_zeros() and saturating_sub(). + + // Shortest version as long as ASCII-ness can be predicted: (especially + // if the BSR instruction which leading_zeros() uses is microcoded or + // doesn't exist) + // u8.leading_zeros() would cast to a bigger type internally, so that's + // free. compensating by shifting left by 24 before inverting lets the + // compiler know that the value passed to leading_zeros() is not zero, + // for which BSR's output is undefined and leading_zeros() normally has + // special case with a branch. + // Shifting one bit too many left acts as a saturating_sub(1). + if self<128 {0} else {((self as u32)<<25).not().leading_zeros() as usize} + + // Branchless but longer version: (9 instructions) + // It's tempting to try (self|0x80).not().leading_zeros().wrapping_sub(1), + // but that produces high lengths for ASCII values 0b01xx_xxxx. + // If we could somehow (branchlessy) clear that bit for ASCII values... + // We can by masking with the value shifted right with sign extension! + // (any nonzero number of bits in range works) + //let extended = self as i8 as i32; + //let ascii_cleared = (extended<<25) & (extended>>25); + //ascii_cleared.not().leading_zeros() as usize + + // cmov version: (7 instructions) + //(((self as u32)<<24).not().leading_zeros() as usize).saturating_sub(1) + } +} + + +/// Methods for working with `u16`s as UTF-16 units. +pub trait U16UtfExt { + /// Will you need an extra unit to complete this codepoint? + /// + /// Returns `Err` for trailing surrogates, `Ok(true)` for leading surrogates, + /// and `Ok(false)` for others. + fn utf16_needs_extra_unit(self) -> Result; + + /// Does this `u16` need another `u16` to complete a codepoint? + /// Returns `(self & 0xfc00) == 0xd800` + /// + /// Is basically an unchecked variant of `utf16_needs_extra_unit()`. + fn is_utf16_leading_surrogate(self) -> bool; +} +impl U16UtfExt for u16 { + #[inline] + fn utf16_needs_extra_unit(self) -> Result { + match self { + // https://en.wikipedia.org/wiki/UTF-16#U.2B10000_to_U.2B10FFFF + 0x00_00...0xd7_ff | 0xe0_00...0xff_ff => Ok(false), + 0xd8_00...0xdb_ff => Ok(true), + _ => Err(InvalidUtf16FirstUnit) + } + } + #[inline] + fn is_utf16_leading_surrogate(self) -> bool { + (self & 0xfc00) == 0xd800// Clear the ten content bytes of a surrogate, + // and see if it's a leading surrogate. + } +} + + + + +/// Extension trait for `char` that adds methods for converting to and from UTF-8 or UTF-16. +pub trait CharExt: Sized { + /// Get the UTF-8 representation of this codepoint. + /// + /// `Utf8Char` is to `[u8;4]` what `char` is to `u32`: + /// a restricted type that cannot be mutated internally. + fn to_utf8(self) -> Utf8Char; + + /// Get the UTF-16 representation of this codepoint. + /// + /// `Utf16Char` is to `[u16;2]` what `char` is to `u32`: + /// a restricted type that cannot be mutated internally. + fn to_utf16(self) -> Utf16Char; + + /// Iterate over or [read](https://doc.rust-lang.org/std/io/trait.Read.html) + /// the one to four bytes in the UTF-8 representation of this codepoint. + /// + /// An identical alternative to the unstable `char.encode_utf8()`. + /// That method somehow still exist on stable, so I have to use a different name. + fn iter_utf8_bytes(self) -> Utf8Iterator; + + /// Iterate over the one or two units in the UTF-16 representation of this codepoint. + /// + /// An identical alternative to the unstable `char.encode_utf16()`. + /// That method somehow still exist on stable, so I have to use a different name. + fn iter_utf16_units(self) -> Utf16Iterator; + + + /// Convert this char to an UTF-8 array, and also return how many bytes of + /// the array are used, + /// + /// The returned array is left-aligned with unused bytes set to zero. + fn to_utf8_array(self) -> ([u8; 4], usize); + + /// Convert this `char` to UTF-16. + /// + /// The second element is non-zero when a surrogate pair is required. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::CharExt; + /// + /// assert_eq!('@'.to_utf16_array(), ['@' as u16, 0]); + /// assert_eq!('睷'.to_utf16_array(), ['睷' as u16, 0]); + /// assert_eq!('\u{abcde}'.to_utf16_array(), [0xda6f, 0xdcde]); + /// ``` + fn to_utf16_array(self) -> [u16; 2]; + + /// Convert this `char` to UTF-16. + /// The second item is `Some` if a surrogate pair is required. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::CharExt; + /// + /// assert_eq!('@'.to_utf16_tuple(), ('@' as u16, None)); + /// assert_eq!('睷'.to_utf16_tuple(), ('睷' as u16, None)); + /// assert_eq!('\u{abcde}'.to_utf16_tuple(), (0xda6f, Some(0xdcde))); + /// ``` + fn to_utf16_tuple(self) -> (u16, Option); + + + + /// Create a `char` from the start of an UTF-8 slice, + /// and also return how many bytes were used. + /// + /// # Errors + /// + /// Returns an `Err` if the slice is empty, doesn't start with a valid + /// UTF-8 sequence or is too short for the sequence. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::CharExt; + /// use encode_unicode::error::InvalidUtf8Slice::*; + /// use encode_unicode::error::InvalidUtf8::*; + /// + /// assert_eq!(char::from_utf8_slice_start(&[b'A', b'B', b'C']), Ok(('A',1))); + /// assert_eq!(char::from_utf8_slice_start(&[0xdd, 0xbb]), Ok(('\u{77b}',2))); + /// + /// assert_eq!(char::from_utf8_slice_start(&[]), Err(TooShort(1))); + /// assert_eq!(char::from_utf8_slice_start(&[0xf0, 0x99]), Err(TooShort(4))); + /// assert_eq!(char::from_utf8_slice_start(&[0xee, b'F', 0x80]), Err(Utf8(NotAContinuationByte(1)))); + /// assert_eq!(char::from_utf8_slice_start(&[0xee, 0x99, 0x0f]), Err(Utf8(NotAContinuationByte(2)))); + /// ``` + fn from_utf8_slice_start(src: &[u8]) -> Result<(Self,usize),InvalidUtf8Slice>; + + /// Create a `char` from the start of an UTF-16 slice, + /// and also return how many units were used. + /// + /// If you want to continue after an error, continue with the next `u16` unit. + fn from_utf16_slice_start(src: &[u16]) -> Result<(Self,usize), InvalidUtf16Slice>; + + + /// Convert an UTF-8 sequence as returned from `.to_utf8_array()` into a `char` + /// + /// The codepoint must start at the first byte, and leftover bytes are ignored. + /// + /// # Errors + /// + /// Returns an `Err` if the array doesn't start with a valid UTF-8 sequence. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::CharExt; + /// use encode_unicode::error::InvalidUtf8Array::*; + /// use encode_unicode::error::InvalidUtf8::*; + /// use encode_unicode::error::InvalidCodepoint::*; + /// + /// assert_eq!(char::from_utf8_array([b'A', 0, 0, 0]), Ok('A')); + /// assert_eq!(char::from_utf8_array([0xf4, 0x8b, 0xbb, 0xbb]), Ok('\u{10befb}')); + /// assert_eq!(char::from_utf8_array([b'A', b'B', b'C', b'D']), Ok('A')); + /// assert_eq!(char::from_utf8_array([0, 0, 0xcc, 0xbb]), Ok('\0')); + /// + /// assert_eq!(char::from_utf8_array([0xef, b'F', 0x80, 0x80]), Err(Utf8(NotAContinuationByte(1)))); + /// assert_eq!(char::from_utf8_array([0xc1, 0x80, 0, 0]), Err(Utf8(OverLong))); + /// assert_eq!(char::from_utf8_array([0xf7, 0xaa, 0x99, 0x88]), Err(Codepoint(TooHigh))); + /// ``` + fn from_utf8_array(utf8: [u8; 4]) -> Result; + + /// Convert a UTF-16 pair as returned from `.to_utf16_array()` into a `char`. + /// + /// The second element is ignored when not required. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::CharExt; + /// use encode_unicode::error::InvalidUtf16Array; + /// + /// assert_eq!(char::from_utf16_array(['x' as u16, 'y' as u16]), Ok('x')); + /// assert_eq!(char::from_utf16_array(['睷' as u16, 0]), Ok('睷')); + /// assert_eq!(char::from_utf16_array([0xda6f, 0xdcde]), Ok('\u{abcde}')); + /// assert_eq!(char::from_utf16_array([0xf111, 0xdbad]), Ok('\u{f111}')); + /// assert_eq!(char::from_utf16_array([0xdaaf, 0xdaaf]), Err(InvalidUtf16Array::SecondIsNotTrailingSurrogate)); + /// assert_eq!(char::from_utf16_array([0xdcac, 0x9000]), Err(InvalidUtf16Array::FirstIsTrailingSurrogate)); + /// ``` + fn from_utf16_array(utf16: [u16; 2]) -> Result; + + /// Convert a UTF-16 pair as returned from `.to_utf16_tuple()` into a `char`. + fn from_utf16_tuple(utf16: (u16, Option)) -> Result; + + + /// Convert an UTF-8 sequence into a char. + /// + /// The length of the slice is taken as length of the sequence; + /// it should be 1,2,3 or 4. + /// + /// # Safety + /// + /// The slice must contain exactly one, valid, UTF-8 sequence. + /// + /// Passing a slice that produces an invalid codepoint is always undefined + /// behavior; Later checks that the codepoint is valid can be removed + /// by the compiler. + /// + /// # Panics + /// + /// If the slice is empty + unsafe fn from_utf8_exact_slice_unchecked(src: &[u8]) -> Self; + + /// Convert a UTF-16 array as returned from `.to_utf16_array()` into a + /// `char`. + /// + /// This function is safe because it avoids creating invalid codepoints, + /// but the returned value might not be what one expectedd. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::CharExt; + /// + /// // starts with a trailing surrogate - converted as if it was a valid + /// // surrogate pair anyway. + /// assert_eq!(char::from_utf16_array_unchecked([0xdbad, 0xf19e]), '\u{fb59e}'); + /// // missing trailing surrogate - ditto + /// assert_eq!(char::from_utf16_array_unchecked([0xd802, 0]), '\u{10800}'); + /// ``` + fn from_utf16_array_unchecked(utf16: [u16;2]) -> Self; + + /// Convert a UTF-16 tuple as returned from `.to_utf16_tuple()` into a `char`. + unsafe fn from_utf16_tuple_unchecked(utf16: (u16, Option)) -> Self; + + + /// Produces more detailed errors than `char::from_u32()` + /// + /// # Errors + /// + /// This function will return an error if + /// + /// * the value is greater than 0x10ffff + /// * the value is between 0xd800 and 0xdfff (inclusive) + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::CharExt; + /// use encode_unicode::error::InvalidCodepoint; + /// + /// assert_eq!(char::from_u32_detailed(0x41), Ok('A')); + /// assert_eq!(char::from_u32_detailed(0x40_00_00), Err(InvalidCodepoint::TooHigh)); + /// assert_eq!(char::from_u32_detailed(0xd951), Err(InvalidCodepoint::Utf16Reserved)); + /// assert_eq!(char::from_u32_detailed(0xdddd), Err(InvalidCodepoint::Utf16Reserved)); + /// assert_eq!(char::from_u32_detailed(0xdd), Ok('Ý')); + /// assert_eq!(char::from_u32_detailed(0x1f331), Ok('🌱')); + /// ``` + fn from_u32_detailed(c: u32) -> Result; +} + + + +impl CharExt for char { + ///////// + //UTF-8// + ///////// + + fn to_utf8(self) -> Utf8Char { + self.into() + } + fn iter_utf8_bytes(self) -> Utf8Iterator { + self.to_utf8().into_iter() + } + + fn to_utf8_array(self) -> ([u8; 4], usize) { + let len = self.len_utf8(); + let mut c = self as u32; + if len == 1 {// ASCII, the common case + ([c as u8, 0, 0, 0], 1) + } else { + let mut parts = 0;// convert to 6-bit bytes + parts |= c & 0x3f; c>>=6; + parts<<=8; parts |= c & 0x3f; c>>=6; + parts<<=8; parts |= c & 0x3f; c>>=6; + parts<<=8; parts |= c & 0x3f; + parts |= 0x80_80_80_80;// set the most significant bit + parts >>= 8*(4-len);// right-align bytes + // Now, unused bytes are zero, (which matters for Utf8Char.eq()) + // and the rest are 0b10xx_xxxx + + // set header on first byte + parts |= (0xff_00u32 >> len) & 0xff;// store length + parts &= Not::not(1u32 << 7-len);// clear the next bit after it + + let bytes: [u8; 4] = unsafe{ mem::transmute(u32::from_le(parts)) }; + (bytes, len) + } + } + + + fn from_utf8_slice_start(src: &[u8]) -> Result<(Self,usize),InvalidUtf8Slice> { + use errors::InvalidUtf8::*; + use errors::InvalidUtf8Slice::*; + let first = match src.first() { + Some(first) => *first, + None => return Err(TooShort(1)), + }; + let bytes = match first.extra_utf8_bytes() { + Err(e) => return Err(Utf8(FirstByte(e))), + Ok(0) => return Ok((first as char, 1)), + Ok(extra) if extra >= src.len() + => return Err(TooShort(extra+1)), + Ok(extra) => &src[..extra+1], + }; + if let Some(i) = bytes.iter().skip(1).position(|&b| (b >> 6) != 0b10 ) { + Err(Utf8(NotAContinuationByte(i+1))) + } else if overlong(bytes[0], bytes[1]) { + Err(Utf8(OverLong)) + } else { + match char::from_u32_detailed(merge_nonascii_unchecked_utf8(bytes)) { + Ok(c) => Ok((c, bytes.len())), + Err(e) => Err(Codepoint(e)), + } + } + } + + fn from_utf8_array(utf8: [u8; 4]) -> Result { + use errors::InvalidUtf8::*; + use errors::InvalidUtf8Array::*; + let src = match utf8[0].extra_utf8_bytes() { + Err(error) => return Err(Utf8(FirstByte(error))), + Ok(0) => return Ok(utf8[0] as char), + Ok(extra) => &utf8[..extra+1], + }; + if let Some(i) = src[1..].iter().position(|&b| (b >> 6) != 0b10 ) { + Err(Utf8(NotAContinuationByte(i+1))) + } else if overlong(utf8[0], utf8[1]) { + Err(Utf8(OverLong)) + } else { + char::from_u32_detailed(merge_nonascii_unchecked_utf8(src)) + .map_err(|e| Codepoint(e) ) + } + } + + unsafe fn from_utf8_exact_slice_unchecked(src: &[u8]) -> Self { + if src.len() == 1 { + src[0] as char + } else { + char::from_u32_unchecked(merge_nonascii_unchecked_utf8(src)) + } + } + + + + ////////// + //UTF-16// + ////////// + + fn to_utf16(self) -> Utf16Char { + Utf16Char::from(self) + } + fn iter_utf16_units(self) -> Utf16Iterator { + self.to_utf16().into_iter() + } + + fn to_utf16_array(self) -> [u16;2] { + let (first, second) = self.to_utf16_tuple(); + [first, second.unwrap_or(0)] + } + fn to_utf16_tuple(self) -> (u16, Option) { + if self <= '\u{ffff}' {// single + (self as u16, None) + } else {// double + let c = self as u32 - 0x_01_00_00; + let high = 0x_d8_00 + (c >> 10); + let low = 0x_dc_00 + (c & 0x_03_ff); + (high as u16, Some(low as u16)) + } + } + + + fn from_utf16_slice_start(src: &[u16]) -> Result<(Self,usize), InvalidUtf16Slice> { + use errors::InvalidUtf16Slice::*; + unsafe {match (src.get(0), src.get(1)) { + (Some(&u @ 0x00_00...0xd7_ff), _) | + (Some(&u @ 0xe0_00...0xff_ff), _) + => Ok((char::from_u32_unchecked(u as u32), 1)), + (Some(&0xdc_00...0xdf_ff), _) => Err(FirstLowSurrogate), + (None, _) => Err(EmptySlice), + (Some(&f @ 0xd8_00...0xdb_ff), Some(&s @ 0xdc_00...0xdf_ff)) + => Ok((char::from_utf16_tuple_unchecked((f, Some(s))), 2)), + (Some(&0xd8_00...0xdb_ff), Some(_)) => Err(SecondNotLowSurrogate), + (Some(&0xd8_00...0xdb_ff), None) => Err(MissingSecond), + (Some(_), _) => unreachable!() + }} + } + + fn from_utf16_array(utf16: [u16;2]) -> Result { + use errors::InvalidUtf16Array::*; + if let Some(c) = char::from_u32(utf16[0] as u32) { + Ok(c) // single + } else if utf16[0] < 0xdc_00 && utf16[1] & 0xfc_00 == 0xdc_00 { + // correct surrogate pair + Ok(combine_surrogates(utf16[0], utf16[1])) + } else if utf16[0] < 0xdc_00 { + Err(SecondIsNotTrailingSurrogate) + } else { + Err(FirstIsTrailingSurrogate) + } + } + fn from_utf16_tuple(utf16: (u16, Option)) -> Result { + use errors::InvalidUtf16Tuple::*; + unsafe{ match utf16 { + (0x00_00...0xd7_ff, None) | // single + (0xe0_00...0xff_ff, None) | // single + (0xd8_00...0xdb_ff, Some(0xdc_00...0xdf_ff)) // correct surrogate + => Ok(char::from_utf16_tuple_unchecked(utf16)), + (0xd8_00...0xdb_ff, Some(_)) => Err(InvalidSecond), + (0xd8_00...0xdb_ff, None ) => Err(MissingSecond), + (0xdc_00...0xdf_ff, _ ) => Err(FirstIsTrailingSurrogate), + ( _ , Some(_)) => Err(SuperfluousSecond), + ( _ , None ) => unreachable!() + }} + } + + fn from_utf16_array_unchecked(utf16: [u16;2]) -> Self { + // treat any array with a surrogate value in [0] as a surrogate because + // combine_surrogates() is safe. + // `(utf16[0] & 0xf800) == 0xd80` might not be quite as fast as + // `utf16[1] != 0`, but avoiding the potential for UB is worth it + // since the conversion isn't zero-cost in either case. + char::from_u32(utf16[0] as u32) + .unwrap_or_else(|| combine_surrogates(utf16[0], utf16[1]) ) + } + unsafe fn from_utf16_tuple_unchecked(utf16: (u16, Option)) -> Self { + match utf16.1 { + Some(second) => combine_surrogates(utf16.0, second), + None => char::from_u32_unchecked(utf16.0 as u32) + } + } + + + fn from_u32_detailed(c: u32) -> Result { + match char::from_u32(c) { + Some(c) => Ok(c), + None if c > 0x10_ff_ff => Err(InvalidCodepoint::TooHigh), + None => Err(InvalidCodepoint::Utf16Reserved), + } + } +} + +// Adapted from https://www.cl.cam.ac.uk/~mgk25/ucs/utf8_check.c +fn overlong(first: u8, second: u8) -> bool { + if first < 0x80 { + false + } else if (first & 0xe0) == 0xc0 { + (first & 0xfe) == 0xc0 + } else if (first & 0xf0) == 0xe0 { + first == 0xe0 && (second & 0xe0) == 0x80 + } else { + first == 0xf0 && (second & 0xf0) == 0x80 + } +} + +/// Decodes the codepoint represented by a multi-byte UTF-8 sequence. +/// +/// Does not check that the codepoint is valid, +/// and returns `u32` because casting invalid codepoints to `char` is insta UB. +fn merge_nonascii_unchecked_utf8(src: &[u8]) -> u32 { + let mut c = src[0] as u32 & (0x7f >> src.len()); + for b in &src[1..] { + c = (c << 6) | (b & 0b0011_1111) as u32; + } + c +} + +/// Create a `char` from a leading and a trailing surrogate. +/// +/// This function is safe because it ignores the six most significant bits of +/// each arguments and always produces a codepoint in 0x01_00_00..=0x10_ff_ff. +fn combine_surrogates(first: u16, second: u16) -> char { + unsafe { + let high = (first & 0x_03_ff) as u32; + let low = (second & 0x_03_ff) as u32; + let c = ((high << 10) | low) + 0x_01_00_00; // no, the constant can't be or'd in + char::from_u32_unchecked(c) + } +} + + + +/// Adds `.utf8chars()` and `.utf16chars()` iterator constructors to `&str`. +pub trait StrExt: AsRef { + /// Equivalent to `.chars()` but produces `Utf8Char`s. + fn utf8chars(&self) -> Utf8Chars; + /// Equivalent to `.chars()` but produces `Utf16Char`s. + fn utf16chars(&self) -> Utf16Chars; + /// Equivalent to `.char_indices()` but produces `Utf8Char`s. + fn utf8char_indices(&self) -> Utf8CharIndices; + /// Equivalent to `.char_indices()` but produces `Utf16Char`s. + fn utf16char_indices(&self) -> Utf16CharIndices; +} + +impl StrExt for str { + fn utf8chars(&self) -> Utf8Chars { + Utf8Chars::from(self) + } + fn utf16chars(&self) -> Utf16Chars { + Utf16Chars::from(self) + } + fn utf8char_indices(&self) -> Utf8CharIndices { + Utf8CharIndices::from(self) + } + fn utf16char_indices(&self) -> Utf16CharIndices { + Utf16CharIndices::from(self) + } +} + +#[cfg(feature="ascii")] +impl StrExt for AsciiStr { + fn utf8chars(&self) -> Utf8Chars { + Utf8Chars::from(self.as_str()) + } + fn utf16chars(&self) -> Utf16Chars { + Utf16Chars::from(self.as_str()) + } + fn utf8char_indices(&self) -> Utf8CharIndices { + Utf8CharIndices::from(self.as_str()) + } + fn utf16char_indices(&self) -> Utf16CharIndices { + Utf16CharIndices::from(self.as_str()) + } +} + + + +/// Iterator methods that convert between `u8`s and `Utf8Char` or `u16`s and `Utf16Char` +/// +/// All the iterator adapters also accept iterators that produce references of +/// the type they convert from. +pub trait IterExt: Iterator+Sized { + /// Converts an iterator of `Utf8Char`s or `&Utf8Char`s to an iterator of + /// `u8`s. + /// + /// Has the same effect as `.flat_map()` or `.flatten()`, but the returned + /// iterator is ~40% faster. + /// + /// The iterator also implements `Read` + /// (when the `std` feature isn't disabled). + /// Reading will never produce an error, and calls to `.read()` and `.next()` + /// can be mixed. + /// + /// The exact number of bytes cannot be known in advance, but `size_hint()` + /// gives the possible range. + /// (min: all remaining characters are ASCII, max: all require four bytes) + /// + /// # Examples + /// + /// From iterator of values: + /// + /// ``` + /// use encode_unicode::{IterExt, StrExt}; + /// + /// let iterator = "foo".utf8chars(); + /// let mut bytes = [0; 4]; + /// for (u,dst) in iterator.to_bytes().zip(&mut bytes) {*dst=u;} + /// assert_eq!(&bytes, b"foo\0"); + /// ``` + /// + /// From iterator of references: + /// + #[cfg_attr(feature="std", doc=" ```")] + #[cfg_attr(not(feature="std"), doc=" ```no_compile")] + /// use encode_unicode::{IterExt, StrExt, Utf8Char}; + /// + /// let chars: Vec = "💣 bomb 💣".utf8chars().collect(); + /// let bytes: Vec = chars.iter().to_bytes().collect(); + /// let flat_map: Vec = chars.iter().flat_map(|u8c| *u8c ).collect(); + /// assert_eq!(bytes, flat_map); + /// ``` + /// + /// `Read`ing from it: + /// + #[cfg_attr(feature="std", doc=" ```")] + #[cfg_attr(not(feature="std"), doc=" ```no_compile")] + /// use encode_unicode::{IterExt, StrExt}; + /// use std::io::Read; + /// + /// let s = "Ååh‽"; + /// assert_eq!(s.len(), 8); + /// let mut buf = [b'E'; 9]; + /// let mut reader = s.utf8chars().to_bytes(); + /// assert_eq!(reader.read(&mut buf[..]).unwrap(), 8); + /// assert_eq!(reader.read(&mut buf[..]).unwrap(), 0); + /// assert_eq!(&buf[..8], s.as_bytes()); + /// assert_eq!(buf[8], b'E'); + /// ``` + fn to_bytes(self) -> Utf8CharSplitter where Self::Item: Borrow; + + /// Converts an iterator of `Utf16Char` (or `&Utf16Char`) to an iterator of + /// `u16`s. + /// + /// Has the same effect as `.flat_map()` or `.flatten()`, but the returned + /// iterator is about twice as fast. + /// + /// The exact number of units cannot be known in advance, but `size_hint()` + /// gives the possible range. + /// + /// # Examples + /// + /// From iterator of values: + /// + /// ``` + /// use encode_unicode::{IterExt, StrExt}; + /// + /// let iterator = "foo".utf16chars(); + /// let mut units = [0; 4]; + /// for (u,dst) in iterator.to_units().zip(&mut units) {*dst=u;} + /// + /// assert_eq!(units, ['f' as u16, 'o' as u16, 'o' as u16, 0]); + /// ``` + /// + /// From iterator of references: + /// + #[cfg_attr(feature="std", doc=" ```")] + #[cfg_attr(not(feature="std"), doc=" ```no_compile")] + /// use encode_unicode::{IterExt, StrExt, Utf16Char}; + /// + /// // (💣 takes two units) + /// let chars: Vec = "💣 bomb 💣".utf16chars().collect(); + /// let units: Vec = chars.iter().to_units().collect(); + /// let flat_map: Vec = chars.iter().flat_map(|u16c| *u16c ).collect(); + /// + /// assert_eq!(units, flat_map); + /// ``` + fn to_units(self) -> Utf16CharSplitter where Self::Item: Borrow; + + /// Decodes bytes as UTF-8 and groups them into `Utf8Char`s + /// + /// When errors (invalid values or sequences) are encountered, + /// it continues with the byte right after the start of the error sequence. + /// This is neither the most intelligent choiche (sometimes it is guaranteed to + /// produce another error), nor the easiest to implement, but I believe it to + /// be the most predictable. + /// It also means that ASCII characters are never hidden by errors. + /// + /// # Examples + /// + /// Replace all errors with u+FFFD REPLACEMENT_CHARACTER: + /// ``` + /// use encode_unicode::{Utf8Char, IterExt}; + /// + /// let mut buf = [b'\0'; 255]; + /// let len = b"foo\xCFbar".iter() + /// .to_utf8chars() + /// .flat_map(|r| r.unwrap_or(Utf8Char::from('\u{FFFD}')).into_iter() ) + /// .zip(&mut buf[..]) + /// .map(|(byte, dst)| *dst = byte ) + /// .count(); + /// + /// assert_eq!(&buf[..len], "foo\u{FFFD}bar".as_bytes()); + /// ``` + /// + /// Collect everything up until the first error into a string: + #[cfg_attr(feature="std", doc=" ```")] + #[cfg_attr(not(feature="std"), doc=" ```no_compile")] + /// use encode_unicode::iterator::Utf8CharMerger; + /// let mut good = String::new(); + /// for r in Utf8CharMerger::from(b"foo\xcc\xbbbar\xcc\xddbaz") { + /// if let Ok(uc) = r { + /// good.push_str(uc.as_str()); + /// } else { + /// break; + /// } + /// } + /// assert_eq!(good, "foo̻bar"); + /// ``` + /// + /// Abort decoding on error: + #[cfg_attr(feature="std", doc=" ```")] + #[cfg_attr(not(feature="std"), doc=" ```no_compile")] + /// use encode_unicode::{IterExt, Utf8Char}; + /// use encode_unicode::error::{InvalidUtf8Slice, InvalidUtf8}; + /// + /// let result = b"ab\0\xe0\xbc\xa9 \xf3\x80\x77".iter() + /// .to_utf8chars() + /// .collect::>(); + /// + /// assert_eq!(result, Err(InvalidUtf8Slice::Utf8(InvalidUtf8::NotAContinuationByte(2)))); + /// ``` + fn to_utf8chars(self) -> Utf8CharMerger where Self::Item: Borrow; + + /// Decodes bytes as UTF-16 and groups them into `Utf16Char`s + /// + /// When errors (unmatched leading surrogates or unexpected trailing surrogates) + /// are encountered, an error is produced for every unit. + /// + /// # Examples + /// + /// Replace errors with '�': + #[cfg_attr(feature="std", doc=" ```")] + #[cfg_attr(not(feature="std"), doc=" ```no_compile")] + /// use encode_unicode::{IterExt, Utf16Char}; + /// + /// let slice = &['a' as u16, 0xdf00, 0xd83c, 0xdca0][..]; + /// let string = slice.iter() + /// .to_utf16chars() + /// .map(|r| r.unwrap_or(Utf16Char::from('\u{fffd}')) ) // REPLACEMENT_CHARACTER + /// .collect::(); + /// + /// assert_eq!(string, "a�🂠"); + /// ``` + /// + /// ``` + /// use encode_unicode::{IterExt, Utf16Char}; + /// use encode_unicode::error::Utf16PairError::*; + /// + /// let slice = [0xdcba, 0xdeff, 0xd8be, 0xdeee, 'Y' as u16, 0xdab1, 0xdab1]; + /// let mut iter = slice.iter().to_utf16chars(); + /// assert_eq!(iter.size_hint(), (3, Some(7))); + /// assert_eq!(iter.next(), Some(Err(UnexpectedTrailingSurrogate))); + /// assert_eq!(iter.next(), Some(Err(UnexpectedTrailingSurrogate))); + /// assert_eq!(iter.next(), Some(Ok(Utf16Char::from('\u{3faee}')))); + /// assert_eq!(iter.next(), Some(Ok(Utf16Char::from('Y')))); + /// assert_eq!(iter.next(), Some(Err(UnmatchedLeadingSurrogate))); + /// assert_eq!(iter.next(), Some(Err(Incomplete))); + /// assert_eq!(iter.into_remaining_units().next(), None); + /// ``` + /// + /// Search for a codepoint and return the codepoint index of the first match: + /// ``` + /// use encode_unicode::{IterExt, Utf16Char}; + /// + /// let position = [0xd875, 0xdd4f, '≈' as u16, '2' as u16].iter() + /// .to_utf16chars() + /// .position(|r| r == Ok(Utf16Char::from('≈')) ); + /// + /// assert_eq!(position, Some(1)); + /// ``` + fn to_utf16chars(self) -> Utf16CharMerger where Self::Item: Borrow; +} + +impl IterExt for I { + fn to_bytes(self) -> Utf8CharSplitter where Self::Item: Borrow { + iter_bytes(self) + } + fn to_units(self) -> Utf16CharSplitter where Self::Item: Borrow { + iter_units(self) + } + fn to_utf8chars(self) -> Utf8CharMerger where Self::Item: Borrow { + Utf8CharMerger::from(self) + } + fn to_utf16chars(self) -> Utf16CharMerger where Self::Item: Borrow { + Utf16CharMerger::from(self) + } +} + + +/// Methods for iterating over `u8` and `u16` slices as UTF-8 or UTF-16 characters. +/// +/// The iterators are slightly faster than the similar methods in [`IterExt`](trait.IterExt.html) +/// because they con "push back" items for free after errors and don't need a +/// separate buffer that must be checked on every call to `.next()`. +pub trait SliceExt: Index { + /// Decode `u8` slices as UTF-8 and iterate over the codepoints as `Utf8Char`s, + /// + /// # Examples + /// + /// Get the index and error type of the first error: + #[cfg_attr(feature="std", doc=" ```")] + #[cfg_attr(not(feature="std"), doc=" ```no_compile")] + /// use encode_unicode::{SliceExt, Utf8Char}; + /// use encode_unicode::error::InvalidUtf8Slice; + /// + /// let slice = b"ab\0\xe0\xbc\xa9 \xf3\x80\x77"; + /// let result = slice.utf8char_indices() + /// .map(|(offset,r,length)| r.map_err(|e| (offset,e,length) ) ) + /// .collect::>(); + /// + /// assert_eq!(result, Err((7, InvalidUtf8Slice::TooShort(4), 1))); + /// ``` + /// + /// ``` + /// use encode_unicode::{SliceExt, Utf8Char}; + /// use std::error::Error; + /// + /// let slice = b"\xf0\xbf\xbf\xbfXY\xdd\xbb\xe1\x80\x99quux123"; + /// let mut fixed_size = [Utf8Char::default(); 8]; + /// for (cp_i, (byte_index, r, _)) in slice.utf8char_indices().enumerate().take(8) { + /// match r { + /// Ok(u8c) => fixed_size[cp_i] = u8c, + /// Err(e) => panic!("Invalid codepoint at index {} ({})", cp_i, e.description()), + /// } + /// } + /// let chars = ['\u{3ffff}', 'X', 'Y', '\u{77b}', '\u{1019}', 'q', 'u', 'u']; + /// assert_eq!(fixed_size, chars); + /// ``` + /// + #[cfg_attr(feature="std", doc=" ```")] + #[cfg_attr(not(feature="std"), doc=" ```no_compile")] + /// use encode_unicode::{SliceExt, Utf8Char}; + /// use encode_unicode::error::InvalidUtf8Slice::*; + /// use encode_unicode::error::{InvalidUtf8, InvalidUtf8FirstByte, InvalidCodepoint}; + /// + /// let bytes = b"\xfa-\xf4\x8f\xee\xa1\x8f-\xed\xa9\x87\xf0\xcc\xbb"; + /// let mut errors = Vec::new(); + /// let mut lengths = Vec::new(); + /// let mut string = String::new(); + /// for (offset,result,length) in bytes.utf8char_indices() { + /// lengths.push((offset,length)); + /// let c = result.unwrap_or_else(|error| { + /// errors.push((offset,error)); + /// Utf8Char::from('\u{fffd}') // replacement character + /// }); + /// string.push_str(c.as_str()); + /// } + /// + /// assert_eq!(string, "�-��\u{e84f}-����\u{33b}"); + /// assert_eq!(lengths, [(0,1), (1,1), (2,1), (3,1), (4,3), (7,1), + /// (8,1), (9,1), (10,1), (11,1), (12,2)]); + /// assert_eq!(errors, [ + /// ( 0, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::TooLongSeqence))), + /// ( 2, Utf8(InvalidUtf8::NotAContinuationByte(2))), + /// ( 3, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::ContinuationByte))), + /// ( 8, Codepoint(InvalidCodepoint::Utf16Reserved)), + /// ( 9, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::ContinuationByte))), + /// (10, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::ContinuationByte))), + /// (11, TooShort(4)), // (but it was not the last element returned!) + /// ]); + /// ``` + fn utf8char_indices(&self) -> Utf8CharDecoder where Self::Output: Borrow<[u8]>; + + + /// Decode `u16` slices as UTF-16 and iterate over the codepoints as `Utf16Char`s, + /// + /// The iterator produces `(usize,Result,usize)`, + /// and the slice is validated as you go. + /// + /// The first `usize` contains the offset from the start of the slice and + /// the last `usize` contains the length of the codepoint or error. + /// The length is either 1 or 2, and always 1 for errors. + /// + /// # Examples + /// + #[cfg_attr(feature="std", doc=" ```")] + #[cfg_attr(not(feature="std"), doc=" ```no_compile")] + /// use encode_unicode::{SliceExt, Utf8Char}; + /// + /// let slice = &['a' as u16, 0xdf00, 0xd83c, 0xdca0][..]; + /// let mut errors = Vec::new(); + /// let string = slice.utf16char_indices().map(|(offset,r,_)| match r { + /// Ok(u16c) => Utf8Char::from(u16c), + /// Err(_) => { + /// errors.push(offset); + /// Utf8Char::from('\u{fffd}') // REPLACEMENT_CHARACTER + /// } + /// }).collect::(); + /// + /// assert_eq!(string, "a�🂠"); + /// assert_eq!(errors, [1]); + /// ``` + /// + /// Search for a codepoint and return its unit and codepoint index. + /// ``` + /// use encode_unicode::{SliceExt, Utf16Char}; + /// + /// let slice = [0xd875,/*'𝕏'*/ 0xdd4f, '≈' as u16, '2' as u16]; + /// let position = slice.utf16char_indices() + /// .enumerate() + /// .find(|&(_,(_,r,_))| r == Ok(Utf16Char::from('≈')) ) + /// .map(|(codepoint, (offset, _, _))| (codepoint, offset) ); + /// + /// assert_eq!(position, Some((1,2))); + /// ``` + /// + /// Error types: + /// ``` + /// use encode_unicode::{SliceExt, Utf16Char}; + /// use encode_unicode::error::Utf16PairError::*; + /// + /// let slice = [0xdcba, 0xdeff, 0xd8be, 0xdeee, 'λ' as u16, 0xdab1, 0xdab1]; + /// let mut iter = slice.utf16char_indices(); + /// assert_eq!(iter.next(), Some((0, Err(UnexpectedTrailingSurrogate), 1))); + /// assert_eq!(iter.next(), Some((1, Err(UnexpectedTrailingSurrogate), 1))); + /// assert_eq!(iter.next(), Some((2, Ok(Utf16Char::from('\u{3faee}')), 2))); + /// assert_eq!(iter.next(), Some((4, Ok(Utf16Char::from('λ')), 1))); + /// assert_eq!(iter.next(), Some((5, Err(UnmatchedLeadingSurrogate), 1))); + /// assert_eq!(iter.next(), Some((6, Err(Incomplete), 1))); + /// assert_eq!(iter.next(), None); + /// assert_eq!(iter.as_slice(), []) + /// ``` + fn utf16char_indices(&self) -> Utf16CharDecoder where Self::Output: Borrow<[u16]>; +} + +impl> SliceExt for S { + fn utf8char_indices(&self) -> Utf8CharDecoder where Self::Output: Borrow<[u8]> { + Utf8CharDecoder::from(self[..].borrow()) + } + fn utf16char_indices(&self) -> Utf16CharDecoder where Self::Output: Borrow<[u16]> { + Utf16CharDecoder::from(self[..].borrow()) + } +} diff --git a/vendor/encode_unicode/src/utf16_char.rs b/vendor/encode_unicode/src/utf16_char.rs new file mode 100644 index 0000000..d8a6cec --- /dev/null +++ b/vendor/encode_unicode/src/utf16_char.rs @@ -0,0 +1,687 @@ +/* Copyright 2016 The encode_unicode Developers + * + * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be + * copied, modified, or distributed except according to those terms. + */ + +use utf16_iterators::Utf16Iterator; +use traits::{CharExt, U16UtfExt}; +use utf8_char::Utf8Char; +use errors::{InvalidUtf16Slice, InvalidUtf16Array, InvalidUtf16Tuple}; +use errors::{NonBMPError, EmptyStrError, FromStrError}; +extern crate core; +use self::core::{hash,fmt}; +use self::core::cmp::Ordering; +use self::core::borrow::Borrow; +use self::core::ops::Deref; +use self::core::str::FromStr; +#[cfg(feature="std")] +use self::core::iter::FromIterator; +#[cfg(feature="std")] +#[allow(deprecated)] +use std::ascii::AsciiExt; +#[cfg(feature="ascii")] +use self::core::char; +#[cfg(feature="ascii")] +extern crate ascii; +#[cfg(feature="ascii")] +use self::ascii::{AsciiChar,ToAsciiChar,ToAsciiCharError}; + + +// I don't think there is any good default value for char, but char does. +#[derive(Default)] +// char doesn't do anything more advanced than u32 for Eq/Ord, so we shouldn't either. +// When it's a single unit, the second is zero, so Eq works. +// #[derive(Ord)] however, breaks on surrogate pairs. +#[derive(PartialEq,Eq)] +#[derive(Clone,Copy)] + + +/// An unicode codepoint stored as UTF-16. +/// +/// It can be borrowed as an `u16` slice, and has the same size as `char`. +pub struct Utf16Char { + units: [u16; 2], +} + + + ///////////////////// + //conversion traits// +///////////////////// +impl FromStr for Utf16Char { + type Err = FromStrError; + /// Create an `Utf16Char` from a string slice. + /// The string must contain exactly one codepoint. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::error::FromStrError::*; + /// use encode_unicode::Utf16Char; + /// use std::str::FromStr; + /// + /// assert_eq!(Utf16Char::from_str("a"), Ok(Utf16Char::from('a'))); + /// assert_eq!(Utf16Char::from_str("🂠"), Ok(Utf16Char::from('🂠'))); + /// assert_eq!(Utf16Char::from_str(""), Err(Empty)); + /// assert_eq!(Utf16Char::from_str("ab"), Err(MultipleCodepoints)); + /// assert_eq!(Utf16Char::from_str("é"), Err(MultipleCodepoints));// 'e'+u301 combining mark + /// ``` + fn from_str(s: &str) -> Result { + match Utf16Char::from_str_start(s) { + Ok((u16c,bytes)) if bytes == s.len() => Ok(u16c), + Ok((_,_)) => Err(FromStrError::MultipleCodepoints), + Err(EmptyStrError) => Err(FromStrError::Empty), + } + } +} +impl From for Utf16Char { + fn from(c: char) -> Self { + let (first, second) = c.to_utf16_tuple(); + Utf16Char{ units: [first, second.unwrap_or(0)] } + } +} +impl From for Utf16Char { + fn from(utf8: Utf8Char) -> Utf16Char { + let (b, utf8_len) = utf8.to_array(); + match utf8_len { + 1 => Utf16Char{ units: [b[0] as u16, 0] }, + 4 => {// need surrogate + let mut first = 0xd800 - (0x01_00_00u32 >> 10) as u16; + first += (b[0] as u16 & 0x07) << 8; + first += (b[1] as u16 & 0x3f) << 2; + first += (b[2] as u16 & 0x30) >> 4; + let mut second = 0xdc00; + second |= (b[2] as u16 & 0x0f) << 6; + second |= b[3] as u16 & 0x3f; + Utf16Char{ units: [first, second] } + }, + _ => { // 2 or 3 + let mut unit = ((b[0] as u16 & 0x1f) << 6) | (b[1] as u16 & 0x3f); + if utf8_len == 3 { + unit = (unit << 6) | (b[2] as u16 & 0x3f); + } + Utf16Char{ units: [unit, 0] } + }, + } + } +} +impl From for char { + fn from(uc: Utf16Char) -> char { + char::from_utf16_array_unchecked(uc.to_array()) + } +} +impl IntoIterator for Utf16Char { + type Item=u16; + type IntoIter=Utf16Iterator; + /// Iterate over the units. + fn into_iter(self) -> Utf16Iterator { + Utf16Iterator::from(self) + } +} + +#[cfg(feature="std")] +impl Extend for Vec { + fn extend>(&mut self, iter: I) { + let iter = iter.into_iter(); + self.reserve(iter.size_hint().0); + for u16c in iter { + self.push(u16c.units[0]); + if u16c.units[1] != 0 { + self.push(u16c.units[1]); + } + } + } +} +#[cfg(feature="std")] +impl<'a> Extend<&'a Utf16Char> for Vec { + fn extend>(&mut self, iter: I) { + self.extend(iter.into_iter().cloned()) + } +} +#[cfg(feature="std")] +impl FromIterator for Vec { + fn from_iter>(iter: I) -> Self { + let mut vec = Vec::new(); + vec.extend(iter); + return vec; + } +} +#[cfg(feature="std")] +impl<'a> FromIterator<&'a Utf16Char> for Vec { + fn from_iter>(iter: I) -> Self { + Self::from_iter(iter.into_iter().cloned()) + } +} + +#[cfg(feature="std")] +impl Extend for String { + fn extend>(&mut self, iter: I) { + self.extend(iter.into_iter().map(|u16c| Utf8Char::from(u16c) )); + } +} +#[cfg(feature="std")] +impl<'a> Extend<&'a Utf16Char> for String { + fn extend>(&mut self, iter: I) { + self.extend(iter.into_iter().cloned()); + } +} +#[cfg(feature="std")] +impl FromIterator for String { + fn from_iter>(iter: I) -> Self { + let mut s = String::new(); + s.extend(iter); + return s; + } +} +#[cfg(feature="std")] +impl<'a> FromIterator<&'a Utf16Char> for String { + fn from_iter>(iter: I) -> Self { + Self::from_iter(iter.into_iter().cloned()) + } +} + + + ///////////////// + //getter traits// +///////////////// +impl AsRef<[u16]> for Utf16Char { + #[inline] + fn as_ref(&self) -> &[u16] { + &self.units[..self.len()] + } +} +impl Borrow<[u16]> for Utf16Char { + #[inline] + fn borrow(&self) -> &[u16] { + self.as_ref() + } +} +impl Deref for Utf16Char { + type Target = [u16]; + #[inline] + fn deref(&self) -> &[u16] { + self.as_ref() + } +} + + + //////////////// + //ascii traits// +//////////////// +#[cfg(feature="std")] +#[allow(deprecated)] +impl AsciiExt for Utf16Char { + type Owned = Self; + fn is_ascii(&self) -> bool { + self.units[0] < 128 + } + fn eq_ignore_ascii_case(&self, other: &Self) -> bool { + self.to_ascii_lowercase() == other.to_ascii_lowercase() + } + fn to_ascii_uppercase(&self) -> Self { + let n = self.units[0].wrapping_sub(b'a' as u16); + if n < 26 {Utf16Char{ units: [n+b'A' as u16, 0] }} + else {*self} + } + fn to_ascii_lowercase(&self) -> Self { + let n = self.units[0].wrapping_sub(b'A' as u16); + if n < 26 {Utf16Char{ units: [n+b'a' as u16, 0] }} + else {*self} + } + fn make_ascii_uppercase(&mut self) { + *self = self.to_ascii_uppercase() + } + fn make_ascii_lowercase(&mut self) { + *self = self.to_ascii_lowercase(); + } +} + +#[cfg(feature="ascii")] +/// Requires the feature "ascii". +impl From for Utf16Char { + #[inline] + fn from(ac: AsciiChar) -> Self { + Utf16Char{ units: [ac.as_byte() as u16, 0] } + } +} +#[cfg(feature="ascii")] +/// Requires the feature "ascii". +impl ToAsciiChar for Utf16Char { + #[inline] + fn to_ascii_char(self) -> Result { + // ToAsciiCHar is not implemented for u16 in ascii 0.9.0 + if self.is_ascii() {self.units[0] as u8} else {255}.to_ascii_char() + } + #[inline] + unsafe fn to_ascii_char_unchecked(self) -> AsciiChar { + (self.units[0] as u8).to_ascii_char_unchecked() + } +} + + + ///////////////////////////////////////////////////////// + //Genaral traits that cannot be derived to emulate char// +///////////////////////////////////////////////////////// +impl hash::Hash for Utf16Char { + fn hash(&self, state: &mut H) { + self.to_char().hash(state); + } +} +impl fmt::Debug for Utf16Char { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self.to_char(), fmtr) + } +} +impl fmt::Display for Utf16Char { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&Utf8Char::from(*self), fmtr) + } +} +// Cannot derive these impls because two-unit characters must always compare +// greater than one-unit ones. +impl PartialOrd for Utf16Char { + #[inline] + fn partial_cmp(&self, rhs: &Self) -> Option { + Some(self.cmp(rhs)) + } +} +impl Ord for Utf16Char { + #[inline] + fn cmp(&self, rhs: &Self) -> Ordering { + // Shift the first unit by 0xd if surrogate, and 0 otherwise. + // This ensures surrogates are always greater than 0xffff, and + // that the second unit only affect the result when the first are equal. + // Multiplying by a constant factor isn't enough because that factor + // would have to be greater than 1023 and smaller than 5.5. + // This transformation is less complicated than combine_surrogates(). + let lhs = (self.units[0] as u32, self.units[1] as u32); + let rhs = (rhs.units[0] as u32, rhs.units[1] as u32); + let lhs = (lhs.0 << (lhs.1 >> 12)) + lhs.1; + let rhs = (rhs.0 << (rhs.1 >> 12)) + rhs.1; + lhs.cmp(&rhs) + } +} + + + //////////////////////////////// + //Comparisons with other types// +//////////////////////////////// +impl PartialEq for Utf16Char { + fn eq(&self, u32c: &char) -> bool { + *self == Utf16Char::from(*u32c) + } +} +impl PartialEq for char { + fn eq(&self, u16c: &Utf16Char) -> bool { + Utf16Char::from(*self) == *u16c + } +} +impl PartialOrd for Utf16Char { + fn partial_cmp(&self, u32c: &char) -> Option { + self.partial_cmp(&Utf16Char::from(*u32c)) + } +} +impl PartialOrd for char { + fn partial_cmp(&self, u16c: &Utf16Char) -> Option { + Utf16Char::from(*self).partial_cmp(u16c) + } +} + +impl PartialEq for Utf16Char { + fn eq(&self, u8c: &Utf8Char) -> bool { + *self == Utf16Char::from(*u8c) + } +} +impl PartialOrd for Utf16Char { + fn partial_cmp(&self, u8c: &Utf8Char) -> Option { + self.partial_cmp(&Utf16Char::from(*u8c)) + } +} +// The other direction is implemented in utf8_char.rs + +/// Only considers the unit equal if the codepoint of the `Utf16Char` is not +/// made up of a surrogate pair. +/// +/// There is no impl in the opposite direction, as this should only be used to +/// compare `Utf16Char`s against constants. +/// +/// # Examples +/// +/// ``` +/// # use encode_unicode::Utf16Char; +/// assert!(Utf16Char::from('6') == b'6' as u16); +/// assert!(Utf16Char::from('\u{FFFF}') == 0xffff_u16); +/// assert!(Utf16Char::from_tuple((0xd876, Some(0xdef9))).unwrap() != 0xd876_u16); +/// ``` +impl PartialEq for Utf16Char { + fn eq(&self, unit: &u16) -> bool { + self.units[0] == *unit && self.units[1] == 0 + } +} +/// Only considers the byte equal if the codepoint of the `Utf16Char` is <= U+FF. +/// +/// # Examples +/// +/// ``` +/// # use encode_unicode::Utf16Char; +/// assert!(Utf16Char::from('6') == b'6'); +/// assert!(Utf16Char::from('\u{00FF}') == b'\xff'); +/// assert!(Utf16Char::from('\u{0100}') != b'\0'); +/// ``` +impl PartialEq for Utf16Char { + fn eq(&self, byte: &u8) -> bool { + self.units[0] == *byte as u16 + } +} +#[cfg(feature = "ascii")] +/// `Utf16Char`s that are not ASCII never compare equal. +impl PartialEq for Utf16Char { + #[inline] + fn eq(&self, ascii: &AsciiChar) -> bool { + self.units[0] == *ascii as u16 + } +} +#[cfg(feature = "ascii")] +/// `Utf16Char`s that are not ASCII never compare equal. +impl PartialEq for AsciiChar { + #[inline] + fn eq(&self, u16c: &Utf16Char) -> bool { + *self as u16 == u16c.units[0] + } +} +#[cfg(feature = "ascii")] +/// `Utf16Char`s that are not ASCII always compare greater. +impl PartialOrd for Utf16Char { + #[inline] + fn partial_cmp(&self, ascii: &AsciiChar) -> Option { + self.units[0].partial_cmp(&(*ascii as u16)) + } +} +#[cfg(feature = "ascii")] +/// `Utf16Char`s that are not ASCII always compare greater. +impl PartialOrd for AsciiChar { + #[inline] + fn partial_cmp(&self, u16c: &Utf16Char) -> Option { + (*self as u16).partial_cmp(&u16c.units[0]) + } +} + + + /////////////////////////////////////////////////////// + //pub impls that should be together for nicer rustdoc// +/////////////////////////////////////////////////////// +impl Utf16Char { + /// Create an `Utf16Char` from the first codepoint in a string slice, + /// converting from UTF-8 to UTF-16. + /// + /// The returned `usize` is the number of UTF-8 bytes used from the str, + /// and not the number of UTF-16 units. + /// + /// Returns an error if the `str` is empty. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::Utf16Char; + /// + /// assert_eq!(Utf16Char::from_str_start("a"), Ok((Utf16Char::from('a'),1))); + /// assert_eq!(Utf16Char::from_str_start("ab"), Ok((Utf16Char::from('a'),1))); + /// assert_eq!(Utf16Char::from_str_start("🂠 "), Ok((Utf16Char::from('🂠'),4))); + /// assert_eq!(Utf16Char::from_str_start("é"), Ok((Utf16Char::from('e'),1)));// 'e'+u301 combining mark + /// assert!(Utf16Char::from_str_start("").is_err()); + /// ``` + pub fn from_str_start(s: &str) -> Result<(Self,usize), EmptyStrError> { + if s.is_empty() { + return Err(EmptyStrError); + } + let b = s.as_bytes(); + // Read the last byte first to reduce the number of unnecesary length checks. + match b[0] { + 0...127 => {// 1 byte => 1 unit + let unit = b[0] as u16;// 0b0000_0000_0xxx_xxxx + Ok((Utf16Char{ units: [unit, 0] }, 1)) + }, + 0b1000_0000...0b1101_1111 => {// 2 bytes => 1 unit + let unit = (((b[1] & 0x3f) as u16) << 0) // 0b0000_0000_00xx_xxxx + | (((b[0] & 0x1f) as u16) << 6);// 0b0000_0xxx_xx00_0000 + Ok((Utf16Char{ units: [unit, 0] }, 2)) + }, + 0b1110_0000...0b1110_1111 => {// 3 bytes => 1 unit + let unit = (((b[2] & 0x3f) as u16) << 0) // 0b0000_0000_00xx_xxxx + | (((b[1] & 0x3f) as u16) << 6) // 0b0000_xxxx_xx00_0000 + | (((b[0] & 0x0f) as u16) << 12);// 0bxxxx_0000_0000_0000 + Ok((Utf16Char{ units: [unit, 0] }, 3)) + }, + _ => {// 4 bytes => 2 units + let second = 0xdc00 // 0b1101_1100_0000_0000 + | (((b[3] & 0x3f) as u16) << 0) // 0b0000_0000_00xx_xxxx + | (((b[2] & 0x0f) as u16) << 6);// 0b0000_00xx_xx00_0000 + let first = 0xd800-(0x01_00_00u32>>10) as u16// 0b1101_0111_1100_0000 + + (((b[2] & 0x30) as u16) >> 4) // 0b0000_0000_0000_00xx + + (((b[1] & 0x3f) as u16) << 2) // 0b0000_0000_xxxx_xx00 + + (((b[0] & 0x07) as u16) << 8); // 0b0000_0xxx_0000_0000 + Ok((Utf16Char{ units: [first, second] }, 4)) + } + } + } + /// Validate and store the first UTF-16 codepoint in the slice. + /// Also return how many units were needed. + pub fn from_slice_start(src: &[u16]) -> Result<(Self,usize), InvalidUtf16Slice> { + char::from_utf16_slice_start(src).map(|(_,len)| { + let second = if len==2 {src[1]} else {0}; + (Utf16Char{ units: [src[0], second] }, len) + }) + } + /// Store the first UTF-16 codepoint of the slice. + /// + /// # Safety + /// + /// The slice must be non-empty and start with a valid UTF-16 codepoint. + /// The length of the slice is never checked. + pub unsafe fn from_slice_start_unchecked(src: &[u16]) -> (Self,usize) { + let first = *src.get_unchecked(0); + if first.is_utf16_leading_surrogate() { + (Utf16Char{ units: [first, *src.get_unchecked(1)] }, 2) + } else { + (Utf16Char{ units: [first, 0] }, 1) + } + } + /// Validate and store an UTF-16 array as returned from `char.to_utf16_array()`. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::Utf16Char; + /// use encode_unicode::error::InvalidUtf16Array; + /// + /// assert_eq!(Utf16Char::from_array(['x' as u16, 'y' as u16]), Ok(Utf16Char::from('x'))); + /// assert_eq!(Utf16Char::from_array(['睷' as u16, 0]), Ok(Utf16Char::from('睷'))); + /// assert_eq!(Utf16Char::from_array([0xda6f, 0xdcde]), Ok(Utf16Char::from('\u{abcde}'))); + /// assert_eq!(Utf16Char::from_array([0xf111, 0xdbad]), Ok(Utf16Char::from('\u{f111}'))); + /// assert_eq!(Utf16Char::from_array([0xdaaf, 0xdaaf]), Err(InvalidUtf16Array::SecondIsNotTrailingSurrogate)); + /// assert_eq!(Utf16Char::from_array([0xdcac, 0x9000]), Err(InvalidUtf16Array::FirstIsTrailingSurrogate)); + /// ``` + pub fn from_array(units: [u16; 2]) -> Result { + if (units[0] & 0xf8_00) != 0xd8_00 { + Ok(Utf16Char { units: [units[0], 0] }) + } else if units[0] < 0xdc_00 && (units[1] & 0xfc_00) == 0xdc_00 { + Ok(Utf16Char { units: units }) + } else if units[0] < 0xdc_00 { + Err(InvalidUtf16Array::SecondIsNotTrailingSurrogate) + } else { + Err(InvalidUtf16Array::FirstIsTrailingSurrogate) + } + } + /// Create an `Utf16Char` from an array as returned from `char.to_utf16_array()`. + /// + /// # Safety + /// + /// The units must form a valid codepoint, and the second unit must be 0 + /// when a surrogate pair is not required. + /// Violating this can easily lead to undefined behavior, although unlike + /// `char` bad `Utf16Char`s simply existing is not immediately UB. + pub unsafe fn from_array_unchecked(units: [u16; 2]) -> Self { + Utf16Char { units: units } + } + /// Validate and store a UTF-16 pair as returned from `char.to_utf16_tuple()`. + pub fn from_tuple(utf16: (u16,Option)) -> Result { + unsafe {char::from_utf16_tuple(utf16).map(|_| + Self::from_tuple_unchecked(utf16) + )} + } + /// Create an `Utf16Char` from a tuple as returned from `char.to_utf16_tuple()`. + /// + /// # Safety + /// + /// The units must form a valid codepoint with the second being 0 when a + /// surrogate pair is not required. + /// Violating this can easily lead to undefined behavior. + pub unsafe fn from_tuple_unchecked(utf16: (u16,Option)) -> Self { + Utf16Char { units: [utf16.0, utf16.1.unwrap_or(0)] } + } + /// Create an `Utf16Char` from a single unit. + /// + /// Codepoints < '\u{1_0000}' (which fit in a `u16`) are part of the basic + /// multilingual plane unless they are reserved for surrogate pairs. + /// + /// # Errors + /// + /// Returns `NonBMPError` if the unit is in the range `0xd800..0xe000` + /// (which means that it's part of a surrogat pair) + /// + /// # Examples + /// + /// ``` + /// # use encode_unicode::Utf16Char; + /// assert_eq!(Utf16Char::from_bmp(0x40).unwrap(), '@'); + /// assert_eq!(Utf16Char::from_bmp('ø' as u16).unwrap(), 'ø'); + /// assert!(Utf16Char::from_bmp(0xdddd).is_err()); + /// ``` + pub fn from_bmp(bmp_codepoint: u16) -> Result { + if bmp_codepoint & 0xf800 != 0xd800 { + Ok(Utf16Char{ units: [bmp_codepoint, 0] }) + } else { + Err(NonBMPError) + } + } + /// Create an `Utf16Char` from a single unit without checking that it's a + /// valid codepoint on its own. + /// + /// # Safety + /// + /// The unit must be less than 0xd800 or greater than 0xdfff. + /// In other words, not part of a surrogate pair. + /// Violating this can easily lead to undefined behavior. + #[inline] + pub unsafe fn from_bmp_unchecked(bmp_codepoint: u16) -> Self { + Utf16Char{ units: [bmp_codepoint, 0] } + } + /// Checks that the codepoint is in the basic multilingual plane. + /// + /// # Examples + /// ``` + /// # use encode_unicode::Utf16Char; + /// assert_eq!(Utf16Char::from('e').is_bmp(), true); + /// assert_eq!(Utf16Char::from('€').is_bmp(), true); + /// assert_eq!(Utf16Char::from('𝔼').is_bmp(), false); + /// ``` + #[inline] + pub fn is_bmp(&self) -> bool { + self.units[1] == 0 + } + + /// The number of units this character is made up of. + /// + /// Is either 1 or 2 and identical to `.as_char().len_utf16()` + /// or `.as_ref().len()`. + #[inline] + pub fn len(self) -> usize { + 1 + (self.units[1] as usize >> 15) + } + // There is no `.is_emty()` because it would always return false. + + /// Checks that the codepoint is an ASCII character. + #[inline] + pub fn is_ascii(&self) -> bool { + self.units[0] <= 127 + } + /// Checks that two characters are an ASCII case-insensitive match. + /// + /// Is equivalent to `a.to_ascii_lowercase() == b.to_ascii_lowercase()`. + #[cfg(feature="std")] + pub fn eq_ignore_ascii_case(&self, other: &Self) -> bool { + self.to_ascii_lowercase() == other.to_ascii_lowercase() + } + /// Converts the character to its ASCII upper case equivalent. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + #[cfg(feature="std")] + pub fn to_ascii_uppercase(&self) -> Self { + let n = self.units[0].wrapping_sub(b'a' as u16); + if n < 26 {Utf16Char{ units: [n+b'A' as u16, 0] }} + else {*self} + } + /// Converts the character to its ASCII lower case equivalent. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + #[cfg(feature="std")] + pub fn to_ascii_lowercase(&self) -> Self { + let n = self.units[0].wrapping_sub(b'A' as u16); + if n < 26 {Utf16Char{ units: [n+b'a' as u16, 0] }} + else {*self} + } + /// Converts the character to its ASCII upper case equivalent in-place. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + #[cfg(feature="std")] + pub fn make_ascii_uppercase(&mut self) { + *self = self.to_ascii_uppercase() + } + /// Converts the character to its ASCII lower case equivalent in-place. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + #[cfg(feature="std")] + pub fn make_ascii_lowercase(&mut self) { + *self = self.to_ascii_lowercase(); + } + + /// Convert from UTF-16 to UTF-32 + pub fn to_char(self) -> char { + self.into() + } + /// Write the internal representation to a slice, + /// and then returns the number of `u16`s written. + /// + /// # Panics + /// Will panic the buffer is too small; + /// You can get the required length from `.len()`, + /// but a buffer of length two is always large enough. + pub fn to_slice(self, dst: &mut[u16]) -> usize { + // Write the last unit first to avoid repeated length checks. + let extra = self.units[1] as usize >> 15; + match dst.get_mut(extra) { + Some(first) => *first = self.units[extra], + None => panic!("The provided buffer is too small.") + } + if extra != 0 {dst[0] = self.units[0];} + extra+1 + } + /// Get the character represented as an array of two units. + /// + /// The second `u16` is zero for codepoints that fit in one unit. + #[inline] + pub fn to_array(self) -> [u16;2] { + self.units + } + /// The second `u16` is used for surrogate pairs. + #[inline] + pub fn to_tuple(self) -> (u16,Option) { + (self.units[0], if self.units[1]==0 {None} else {Some(self.units[1])}) + } +} diff --git a/vendor/encode_unicode/src/utf16_iterators.rs b/vendor/encode_unicode/src/utf16_iterators.rs new file mode 100644 index 0000000..7adb5ac --- /dev/null +++ b/vendor/encode_unicode/src/utf16_iterators.rs @@ -0,0 +1,270 @@ +/* Copyright 2016 The encode_unicode Developers + * + * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be + * copied, modified, or distributed except according to those terms. + */ + +use traits::CharExt; +use utf16_char::Utf16Char; +use errors::EmptyStrError; +extern crate core; +use self::core::fmt; +use self::core::borrow::Borrow; + +// Invalid values that says the field is consumed or empty. +const FIRST_USED: u16 = 0x_dc_00; +const SECOND_USED: u16 = 0; + +/// Iterate over the units of the UTF-16 representation of a codepoint. +#[derive(Clone)] +pub struct Utf16Iterator { + first: u16, + second: u16, +} +impl From for Utf16Iterator { + fn from(c: char) -> Self { + let (first, second) = c.to_utf16_tuple(); + Utf16Iterator{ first: first, second: second.unwrap_or(SECOND_USED) } + } +} +impl From for Utf16Iterator { + fn from(uc: Utf16Char) -> Self { + let (first, second) = uc.to_tuple(); + Utf16Iterator{ first: first, second: second.unwrap_or(SECOND_USED) } + } +} +impl Iterator for Utf16Iterator { + type Item=u16; + fn next(&mut self) -> Option { + match (self.first, self.second) { + (FIRST_USED, SECOND_USED) => { None }, + (FIRST_USED, second ) => {self.second = SECOND_USED; Some(second)}, + (first , _ ) => {self.first = FIRST_USED; Some(first )}, + } + } + fn size_hint(&self) -> (usize, Option) { + (self.len(), Some(self.len())) + } +} +impl ExactSizeIterator for Utf16Iterator { + fn len(&self) -> usize { + (if self.first == FIRST_USED {0} else {1}) + + (if self.second == SECOND_USED {0} else {1}) + } +} +impl fmt::Debug for Utf16Iterator { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + let mut clone = self.clone(); + match (clone.next(), clone.next()) { + (Some(one), None) => write!(fmtr, "[{}]", one), + (Some(a), Some(b)) => write!(fmtr, "[{}, {}]", a, b), + (None, _) => write!(fmtr, "[]"), + } + } +} + + + +/// Converts an iterator of `Utf16Char` (or `&Utf16Char`) +/// to an iterator of `u16`s. +/// Is equivalent to calling `.flat_map()` on the original iterator, +/// but the returned iterator is about twice as fast. +/// +/// The exact number of units cannot be known in advance, but `size_hint()` +/// gives the possible range. +/// +/// # Examples +/// +/// From iterator of values: +/// +/// ``` +/// use encode_unicode::{iter_units, CharExt}; +/// +/// let iterator = "foo".chars().map(|c| c.to_utf16() ); +/// let mut units = [0; 4]; +/// for (u,dst) in iter_units(iterator).zip(&mut units) {*dst=u;} +/// assert_eq!(units, ['f' as u16, 'o' as u16, 'o' as u16, 0]); +/// ``` +/// +/// From iterator of references: +/// +#[cfg_attr(feature="std", doc=" ```")] +#[cfg_attr(not(feature="std"), doc=" ```no_compile")] +/// use encode_unicode::{iter_units, CharExt, Utf16Char}; +/// +/// // (💣 takes two units) +/// let chars: Vec = "💣 bomb 💣".chars().map(|c| c.to_utf16() ).collect(); +/// let units: Vec = iter_units(&chars).collect(); +/// let flat_map: Vec = chars.iter().flat_map(|u16c| *u16c ).collect(); +/// assert_eq!(units, flat_map); +/// ``` +pub fn iter_units, I:IntoIterator> +(iterable: I) -> Utf16CharSplitter { + Utf16CharSplitter{ inner: iterable.into_iter(), prev_second: 0 } +} + +/// The iterator type returned by `iter_units()` +#[derive(Clone)] +pub struct Utf16CharSplitter, I:Iterator> { + inner: I, + prev_second: u16, +} +impl> From for Utf16CharSplitter { + /// A less generic constructor than `iter_units()` + fn from(iter: I) -> Self { + iter_units(iter) + } +} +impl, I:Iterator> Utf16CharSplitter { + /// Extracts the source iterator. + /// + /// Note that `iter_units(iter.into_inner())` is not a no-op: + /// If the last returned unit from `next()` was a leading surrogate, + /// the trailing surrogate is lost. + pub fn into_inner(self) -> I { + self.inner + } +} +impl, I:Iterator> Iterator for Utf16CharSplitter { + type Item = u16; + fn next(&mut self) -> Option { + if self.prev_second == 0 { + self.inner.next().map(|u16c| { + let units = u16c.borrow().to_array(); + self.prev_second = units[1]; + units[0] + }) + } else { + let prev_second = self.prev_second; + self.prev_second = 0; + Some(prev_second) + } + } + fn size_hint(&self) -> (usize,Option) { + // Doesn't need to handle unlikely overflows correctly because + // size_hint() cannot be relied upon anyway. (the trait isn't unsafe) + let (min, max) = self.inner.size_hint(); + let add = if self.prev_second == 0 {0} else {1}; + (min.wrapping_add(add), max.map(|max| max.wrapping_mul(2).wrapping_add(add) )) + } +} + + + +/// An iterator over the codepoints in a `str` represented as `Utf16Char`. +#[derive(Clone)] +pub struct Utf16CharIndices<'a>{ + str: &'a str, + index: usize, +} +impl<'a> From<&'a str> for Utf16CharIndices<'a> { + fn from(s: &str) -> Utf16CharIndices { + Utf16CharIndices{str: s, index: 0} + } +} +impl<'a> Utf16CharIndices<'a> { + /// Extract the remainder of the source `str`. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::{StrExt, Utf16Char}; + /// let mut iter = "abc".utf16char_indices(); + /// assert_eq!(iter.next_back(), Some((2, Utf16Char::from('c')))); + /// assert_eq!(iter.next(), Some((0, Utf16Char::from('a')))); + /// assert_eq!(iter.as_str(), "b"); + /// ``` + pub fn as_str(&self) -> &'a str { + &self.str[self.index..] + } +} +impl<'a> Iterator for Utf16CharIndices<'a> { + type Item = (usize,Utf16Char); + fn next(&mut self) -> Option<(usize,Utf16Char)> { + match Utf16Char::from_str_start(&self.str[self.index..]) { + Ok((u16c, bytes)) => { + let item = (self.index, u16c); + self.index += bytes; + Some(item) + }, + Err(EmptyStrError) => None + } + } + fn size_hint(&self) -> (usize,Option) { + let len = self.str.len() - self.index; + // For len+3 to overflow, the slice must fill all but two bytes of + // addressable memory, and size_hint() doesn't need to be correct. + (len.wrapping_add(3)/4, Some(len)) + } +} +impl<'a> DoubleEndedIterator for Utf16CharIndices<'a> { + fn next_back(&mut self) -> Option<(usize,Utf16Char)> { + if self.index < self.str.len() { + let rev = self.str.bytes().rev(); + let len = 1 + rev.take_while(|b| b & 0b1100_0000 == 0b1000_0000 ).count(); + let starts = self.str.len() - len; + let (u16c,_) = Utf16Char::from_str_start(&self.str[starts..]).unwrap(); + self.str = &self.str[..starts]; + Some((starts, u16c)) + } else { + None + } + } +} +impl<'a> fmt::Debug for Utf16CharIndices<'a> { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + fmtr.debug_tuple("Utf16CharIndices") + .field(&self.index) + .field(&self.as_str()) + .finish() + } +} + + +/// An iterator over the codepoints in a `str` represented as `Utf16Char`. +#[derive(Clone)] +pub struct Utf16Chars<'a>(Utf16CharIndices<'a>); +impl<'a> From<&'a str> for Utf16Chars<'a> { + fn from(s: &str) -> Utf16Chars { + Utf16Chars(Utf16CharIndices::from(s)) + } +} +impl<'a> Utf16Chars<'a> { + /// Extract the remainder of the source `str`. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::{StrExt, Utf16Char}; + /// let mut iter = "abc".utf16chars(); + /// assert_eq!(iter.next(), Some(Utf16Char::from('a'))); + /// assert_eq!(iter.next_back(), Some(Utf16Char::from('c'))); + /// assert_eq!(iter.as_str(), "b"); + /// ``` + pub fn as_str(&self) -> &'a str { + self.0.as_str() + } +} +impl<'a> Iterator for Utf16Chars<'a> { + type Item = Utf16Char; + fn next(&mut self) -> Option { + self.0.next().map(|(_,u16c)| u16c ) + } + fn size_hint(&self) -> (usize,Option) { + self.0.size_hint() + } +} +impl<'a> DoubleEndedIterator for Utf16Chars<'a> { + fn next_back(&mut self) -> Option { + self.0.next_back().map(|(_,u16c)| u16c ) + } +} +impl<'a> fmt::Debug for Utf16Chars<'a> { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + fmtr.debug_tuple("Utf16Chars") + .field(&self.as_str()) + .finish() + } +} diff --git a/vendor/encode_unicode/src/utf8_char.rs b/vendor/encode_unicode/src/utf8_char.rs new file mode 100644 index 0000000..9dba4ba --- /dev/null +++ b/vendor/encode_unicode/src/utf8_char.rs @@ -0,0 +1,647 @@ +/* Copyright 2016 The encode_unicode Developers + * + * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be + * copied, modified, or distributed except according to those terms. + */ + +use errors::{FromStrError, EmptyStrError, NonAsciiError, InvalidUtf8Slice, InvalidUtf8Array}; +use utf8_iterators::Utf8Iterator; +use traits::{CharExt, U8UtfExt}; +use utf16_char::Utf16Char; +extern crate core; +use self::core::{hash, fmt, str, ptr}; +use self::core::cmp::Ordering; +use self::core::borrow::Borrow; +use self::core::ops::Deref; +use self::core::mem::transmute; +#[cfg(feature="std")] +use self::core::iter::FromIterator; +#[cfg(feature="std")] +#[allow(deprecated)] +use std::ascii::AsciiExt; +#[cfg(feature="ascii")] +extern crate ascii; +#[cfg(feature="ascii")] +use self::ascii::{AsciiChar,ToAsciiChar,ToAsciiCharError}; + + +// I don't think there is any good default value for char, but char does. +#[derive(Default)] +// char doesn't do anything more advanced than u32 for Eq/Ord, so we shouldn't either. +// The default impl of Ord for arrays works out because longer codepoints +// start with more ones, so if they're equal, the length is the same, +// breaks down for values above 0x1f_ff_ff but those can only be created by unsafe code. +#[derive(PartialEq,Eq, PartialOrd,Ord)] + +#[derive(Clone,Copy)] + + +/// An unicode codepoint stored as UTF-8. +/// +/// It can be borrowed as a `str`, and has the same size as `char`. +pub struct Utf8Char { + bytes: [u8; 4], +} + + + ///////////////////// + //conversion traits// +///////////////////// +impl str::FromStr for Utf8Char { + type Err = FromStrError; + /// Create an `Utf8Char` from a string slice. + /// The string must contain exactly one codepoint. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::error::FromStrError::*; + /// use encode_unicode::Utf8Char; + /// use std::str::FromStr; + /// + /// assert_eq!(Utf8Char::from_str("a"), Ok(Utf8Char::from('a'))); + /// assert_eq!(Utf8Char::from_str("🂠"), Ok(Utf8Char::from('🂠'))); + /// assert_eq!(Utf8Char::from_str(""), Err(Empty)); + /// assert_eq!(Utf8Char::from_str("ab"), Err(MultipleCodepoints)); + /// assert_eq!(Utf8Char::from_str("é"), Err(MultipleCodepoints));// 'e'+u301 combining mark + /// ``` + fn from_str(s: &str) -> Result { + if s.is_empty() { + Err(FromStrError::Empty) + } else if s.len() != 1+s.as_bytes()[0].extra_utf8_bytes_unchecked() { + Err(FromStrError::MultipleCodepoints) + } else { + let mut bytes = [0; 4]; + bytes[..s.len()].copy_from_slice(s.as_bytes()); + Ok(Utf8Char{bytes: bytes}) + } + } +} +impl From for Utf8Char { + fn from(utf16: Utf16Char) -> Utf8Char { + match utf16.to_tuple() { + (a @ 0...0x00_7f, _) => { + Utf8Char{ bytes: [a as u8, 0, 0, 0] } + }, + (u @ 0...0x07_ff, _) => { + let b = 0x80 | (u & 0x00_3f) as u8; + let a = 0xc0 | ((u & 0x07_c0) >> 6) as u8; + Utf8Char{ bytes: [a, b, 0, 0] } + }, + (u, None) => { + let c = 0x80 | (u & 0x00_3f) as u8; + let b = 0x80 | ((u & 0x0f_c0) >> 6) as u8; + let a = 0xe0 | ((u & 0xf0_00) >> 12) as u8; + Utf8Char{ bytes: [a, b, c, 0] } + }, + (f, Some(s)) => { + let f = f + (0x01_00_00u32 >> 10) as u16; + let d = 0x80 | (s & 0x00_3f) as u8; + let c = 0x80 | ((s & 0x03_c0) >> 6) as u8 + | ((f & 0x00_03) << 4) as u8; + let b = 0x80 | ((f & 0x00_fc) >> 2) as u8; + let a = 0xf0 | ((f & 0x07_00) >> 8) as u8; + Utf8Char{ bytes: [a, b, c, d] } + } + } + } +} +impl From for Utf8Char { + fn from(c: char) -> Self { + Utf8Char{ bytes: c.to_utf8_array().0 } + } +} +impl From for char { + fn from(uc: Utf8Char) -> char { + unsafe{ char::from_utf8_exact_slice_unchecked(&uc.bytes[..uc.len()]) } + } +} +impl IntoIterator for Utf8Char { + type Item=u8; + type IntoIter=Utf8Iterator; + /// Iterate over the byte values. + fn into_iter(self) -> Utf8Iterator { + Utf8Iterator::from(self) + } +} + +#[cfg(feature="std")] +impl Extend for Vec { + fn extend>(&mut self, iter: I) { + let iter = iter.into_iter(); + self.reserve(iter.size_hint().0); + for u8c in iter { + // twice as fast as self.extend_from_slice(u8c.as_bytes()); + self.push(u8c.bytes[0]); + for &extra in &u8c.bytes[1..] { + if extra != 0 { + self.push(extra); + } + } + } + } +} +#[cfg(feature="std")] +impl<'a> Extend<&'a Utf8Char> for Vec { + fn extend>(&mut self, iter: I) { + self.extend(iter.into_iter().cloned()) + } +} +#[cfg(feature="std")] +impl Extend for String { + fn extend>(&mut self, iter: I) { + unsafe { self.as_mut_vec().extend(iter) } + } +} +#[cfg(feature="std")] +impl<'a> Extend<&'a Utf8Char> for String { + fn extend>(&mut self, iter: I) { + self.extend(iter.into_iter().cloned()) + } +} +#[cfg(feature="std")] +impl FromIterator for String { + fn from_iter>(iter: I) -> String { + let mut string = String::new(); + string.extend(iter); + return string; + } +} +#[cfg(feature="std")] +impl<'a> FromIterator<&'a Utf8Char> for String { + fn from_iter>(iter: I) -> String { + iter.into_iter().cloned().collect() + } +} +#[cfg(feature="std")] +impl FromIterator for Vec { + fn from_iter>(iter: I) -> Self { + iter.into_iter().collect::().into_bytes() + } +} +#[cfg(feature="std")] +impl<'a> FromIterator<&'a Utf8Char> for Vec { + fn from_iter>(iter: I) -> Self { + iter.into_iter().cloned().collect::().into_bytes() + } +} + + + ///////////////// + //getter traits// +///////////////// +impl AsRef<[u8]> for Utf8Char { + fn as_ref(&self) -> &[u8] { + &self.bytes[..self.len()] + } +} +impl AsRef for Utf8Char { + fn as_ref(&self) -> &str { + unsafe{ str::from_utf8_unchecked( self.as_ref() ) } + } +} +impl Borrow<[u8]> for Utf8Char { + fn borrow(&self) -> &[u8] { + self.as_ref() + } +} +impl Borrow for Utf8Char { + fn borrow(&self) -> &str { + self.as_ref() + } +} +impl Deref for Utf8Char { + type Target = str; + fn deref(&self) -> &Self::Target { + self.as_ref() + } +} + + + //////////////// + //ascii traits// +//////////////// +#[cfg(feature="std")] +#[allow(deprecated)] +impl AsciiExt for Utf8Char { + type Owned = Utf8Char; + fn is_ascii(&self) -> bool { + self.bytes[0].is_ascii() + } + fn eq_ignore_ascii_case(&self, other: &Self) -> bool { + if self.is_ascii() {self.bytes[0].eq_ignore_ascii_case(&other.bytes[0])} + else {self == other} + } + fn to_ascii_uppercase(&self) -> Self::Owned { + let mut uc = *self; + uc.make_ascii_uppercase(); + uc + } + fn to_ascii_lowercase(&self) -> Self::Owned { + let mut uc = *self; + uc.make_ascii_lowercase(); + uc + } + fn make_ascii_uppercase(&mut self) { + self.bytes[0].make_ascii_uppercase() + } + fn make_ascii_lowercase(&mut self) { + self.bytes[0].make_ascii_lowercase(); + } +} + +#[cfg(feature="ascii")] +/// Requires the feature "ascii". +impl From for Utf8Char { + fn from(ac: AsciiChar) -> Self { + Utf8Char{ bytes: [ac.as_byte(),0,0,0] } + } +} +#[cfg(feature="ascii")] +/// Requires the feature "ascii". +impl ToAsciiChar for Utf8Char { + fn to_ascii_char(self) -> Result { + self.bytes[0].to_ascii_char() + } + unsafe fn to_ascii_char_unchecked(self) -> AsciiChar { + self.bytes[0].to_ascii_char_unchecked() + } +} + + + ///////////////////////////////////////////////////////// + //Genaral traits that cannot be derived to emulate char// +///////////////////////////////////////////////////////// +impl hash::Hash for Utf8Char { + fn hash(&self, state: &mut H) { + self.to_char().hash(state); + } +} +impl fmt::Debug for Utf8Char { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self.to_char(), fmtr) + } +} +impl fmt::Display for Utf8Char { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + fmtr.write_str(self.as_str()) + } +} + + + //////////////////////////////// + //Comparisons with other types// +//////////////////////////////// +impl PartialEq for Utf8Char { + fn eq(&self, u32c: &char) -> bool { + *self == Utf8Char::from(*u32c) + } +} +impl PartialEq for char { + fn eq(&self, u8c: &Utf8Char) -> bool { + Utf8Char::from(*self) == *u8c + } +} +impl PartialOrd for Utf8Char { + fn partial_cmp(&self, u32c: &char) -> Option { + self.partial_cmp(&Self::from(*u32c)) + } +} +impl PartialOrd for char { + fn partial_cmp(&self, u8c: &Utf8Char) -> Option { + Utf8Char::from(*self).partial_cmp(u8c) + } +} + +impl PartialEq for Utf8Char { + fn eq(&self, u16c: &Utf16Char) -> bool { + *self == Self::from(*u16c) + } +} +impl PartialOrd for Utf8Char { + fn partial_cmp(&self, u16c: &Utf16Char) -> Option { + self.partial_cmp(&Self::from(*u16c)) + } +} +// The other direction is implemented in utf16_char.rs + +/// Only considers the byte equal if both it and the `Utf8Char` represents ASCII characters. +/// +/// There is no impl in the opposite direction, as this should only be used to +/// compare `Utf8Char`s against constants. +/// +/// # Examples +/// +/// ``` +/// # use encode_unicode::Utf8Char; +/// assert!(Utf8Char::from('8') == b'8'); +/// assert!(Utf8Char::from_array([0xf1,0x80,0x80,0x80]).unwrap() != 0xf1); +/// assert!(Utf8Char::from('\u{ff}') != 0xff); +/// assert!(Utf8Char::from('\u{80}') != 0x80); +/// ``` +impl PartialEq for Utf8Char { + fn eq(&self, byte: &u8) -> bool { + self.bytes[0] == *byte && self.bytes[1] == 0 + } +} +#[cfg(feature = "ascii")] +/// `Utf8Char`s that are not ASCII never compare equal. +impl PartialEq for Utf8Char { + #[inline] + fn eq(&self, ascii: &AsciiChar) -> bool { + self.bytes[0] == *ascii as u8 + } +} +#[cfg(feature = "ascii")] +/// `Utf8Char`s that are not ASCII never compare equal. +impl PartialEq for AsciiChar { + #[inline] + fn eq(&self, u8c: &Utf8Char) -> bool { + u8c == self + } +} +#[cfg(feature = "ascii")] +/// `Utf8Char`s that are not ASCII always compare greater. +impl PartialOrd for Utf8Char { + #[inline] + fn partial_cmp(&self, ascii: &AsciiChar) -> Option { + self.bytes[0].partial_cmp(ascii) + } +} +#[cfg(feature = "ascii")] +/// `Utf8Char`s that are not ASCII always compare greater. +impl PartialOrd for AsciiChar { + #[inline] + fn partial_cmp(&self, u8c: &Utf8Char) -> Option { + self.partial_cmp(&u8c.bytes[0]) + } +} + + + /////////////////////////////////////////////////////// + //pub impls that should be together for nicer rustdoc// +/////////////////////////////////////////////////////// +impl Utf8Char { + /// Create an `Utf8Char` from the first codepoint in a `str`. + /// + /// Returns an error if the `str` is empty. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::Utf8Char; + /// + /// assert_eq!(Utf8Char::from_str_start("a"), Ok((Utf8Char::from('a'),1))); + /// assert_eq!(Utf8Char::from_str_start("ab"), Ok((Utf8Char::from('a'),1))); + /// assert_eq!(Utf8Char::from_str_start("🂠 "), Ok((Utf8Char::from('🂠'),4))); + /// assert_eq!(Utf8Char::from_str_start("é"), Ok((Utf8Char::from('e'),1)));// 'e'+u301 combining mark + /// assert!(Utf8Char::from_str_start("").is_err()); + /// ``` + pub fn from_str_start(src: &str) -> Result<(Self,usize),EmptyStrError> { + unsafe { + if src.is_empty() { + Err(EmptyStrError) + } else { + Ok(Utf8Char::from_slice_start_unchecked(src.as_bytes())) + } + } + } + /// Create an `Utf8Char` of the first codepoint in an UTF-8 slice. + /// Also returns the length of the UTF-8 sequence for the codepoint. + /// + /// If the slice is from a `str`, use `::from_str_start()` to skip UTF-8 validation. + /// + /// # Errors + /// + /// Returns an `Err` if the slice is empty, doesn't start with a valid + /// UTF-8 sequence or is too short for the sequence. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::Utf8Char; + /// use encode_unicode::error::InvalidUtf8Slice::*; + /// use encode_unicode::error::InvalidUtf8::*; + /// + /// assert_eq!(Utf8Char::from_slice_start(&[b'A', b'B', b'C']), Ok((Utf8Char::from('A'),1))); + /// assert_eq!(Utf8Char::from_slice_start(&[0xdd, 0xbb]), Ok((Utf8Char::from('\u{77b}'),2))); + /// + /// assert_eq!(Utf8Char::from_slice_start(&[]), Err(TooShort(1))); + /// assert_eq!(Utf8Char::from_slice_start(&[0xf0, 0x99]), Err(TooShort(4))); + /// assert_eq!(Utf8Char::from_slice_start(&[0xee, b'F', 0x80]), Err(Utf8(NotAContinuationByte(1)))); + /// assert_eq!(Utf8Char::from_slice_start(&[0xee, 0x99, 0x0f]), Err(Utf8(NotAContinuationByte(2)))); + /// ``` + pub fn from_slice_start(src: &[u8]) -> Result<(Self,usize),InvalidUtf8Slice> { + char::from_utf8_slice_start(src).map(|(_,len)| { + let mut bytes = [0; 4]; + bytes[..len].copy_from_slice(&src[..len]); + (Utf8Char{ bytes: bytes }, len) + }) + } + /// A `from_slice_start()` that doesn't validate the codepoint. + /// + /// # Safety + /// + /// The slice must be non-empty and start with a valid UTF-8 codepoint. + /// Invalid or incomplete values might cause reads of uninitalized memory. + pub unsafe fn from_slice_start_unchecked(src: &[u8]) -> (Self,usize) { + let len = 1+src.get_unchecked(0).extra_utf8_bytes_unchecked(); + let mut bytes = [0; 4]; + ptr::copy_nonoverlapping(src.as_ptr(), &mut bytes[0] as *mut u8, len); + (Utf8Char{ bytes: bytes }, len) + } + /// Create an `Utf8Char` from a byte array after validating it. + /// + /// The codepoint must start at the first byte. + /// Unused bytes are set to zero by this function and so can be anything. + /// + /// # Errors + /// + /// Returns an `Err` if the array doesn't start with a valid UTF-8 sequence. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::Utf8Char; + /// use encode_unicode::error::InvalidUtf8Array::*; + /// use encode_unicode::error::InvalidUtf8::*; + /// use encode_unicode::error::InvalidCodepoint::*; + /// + /// assert_eq!(Utf8Char::from_array([b'A', 0, 0, 0]), Ok(Utf8Char::from('A'))); + /// assert_eq!(Utf8Char::from_array([0xf4, 0x8b, 0xbb, 0xbb]), Ok(Utf8Char::from('\u{10befb}'))); + /// assert_eq!(Utf8Char::from_array([b'A', b'B', b'C', b'D']), Ok(Utf8Char::from('A'))); + /// assert_eq!(Utf8Char::from_array([0, 0, 0xcc, 0xbb]), Ok(Utf8Char::from('\0'))); + /// + /// assert_eq!(Utf8Char::from_array([0xef, b'F', 0x80, 0x80]), Err(Utf8(NotAContinuationByte(1)))); + /// assert_eq!(Utf8Char::from_array([0xc1, 0x80, 0, 0]), Err(Utf8(OverLong))); + /// assert_eq!(Utf8Char::from_array([0xf7, 0xaa, 0x99, 0x88]), Err(Codepoint(TooHigh))); + /// ``` + pub fn from_array(utf8: [u8;4]) -> Result { + unsafe { + // perform all validation + try!(char::from_utf8_array(utf8)); + let extra = utf8[0].extra_utf8_bytes_unchecked() as u32; + // zero unused bytes in one operation by transmuting the arrary to + // u32, apply an endian-corrected mask and transmute back + let mask = u32::from_le(0xff_ff_ff_ff >> 8*(3-extra)); + let unused_zeroed = mask & transmute::<_,u32>(utf8); + Ok(Utf8Char{ bytes: transmute(unused_zeroed) }) + } + } + /// Zero-cost constructor. + /// + /// # Safety + /// + /// Must contain a valid codepoint starting at the first byte, with the + /// unused bytes zeroed. + /// Bad values can easily lead to undefined behavior. + #[inline] + pub unsafe fn from_array_unchecked(utf8: [u8;4]) -> Self { + Utf8Char{ bytes: utf8 } + } + /// Create an `Utf8Char` from a single byte. + /// + /// The byte must be an ASCII character. + /// + /// # Errors + /// + /// Returns `NonAsciiError` if the byte greater than 127. + /// + /// # Examples + /// + /// ``` + /// # use encode_unicode::Utf8Char; + /// assert_eq!(Utf8Char::from_ascii(b'a').unwrap(), 'a'); + /// assert!(Utf8Char::from_ascii(128).is_err()); + /// ``` + pub fn from_ascii(ascii: u8) -> Result { + if ascii as i8 >= 0 { + Ok(Utf8Char{ bytes: [ascii, 0, 0, 0] }) + } else { + Err(NonAsciiError) + } + } + /// Create an `Utf8Char` from a single byte without checking that it's a + /// valid codepoint on its own, which is only true for ASCII characters. + /// + /// # Safety + /// + /// The byte must be less than 128. + #[inline] + pub unsafe fn from_ascii_unchecked(ascii: u8) -> Self { + Utf8Char{ bytes: [ascii, 0, 0, 0] } + } + + /// The number of bytes this character needs. + /// + /// Is between 1 and 4 (inclusive) and identical to `.as_ref().len()` or + /// `.as_char().len_utf8()`. + #[inline] + pub fn len(self) -> usize { + // Invariants of the extra bytes enambles algorithms that + // `u8.extra_utf8_bytes_unchecked()` cannot use. + // Some of them turned out to require fewer x86 instructions: + + // Exploits that unused bytes are zero and calculates the number of + // trailing zero bytes. + // Setting a bit in the first byte prevents the function from returning + // 0 for '\0' (which has 32 leading zeros). + // trailing and leading is swapped below to optimize for little-endian + // architectures. + (4 - (u32::to_le(unsafe{transmute(self.bytes)})|1).leading_zeros()/8) as usize + + // Exploits that the extra bytes have their most significant bit set if + // in use. + // Takes fewer instructions than the one above if popcnt can be used, + // (which it cannot by default, + // set RUSTFLAGS='-C target-cpu=native' to enable) + //let all: u32 = unsafe{transmute(self.bytes)}; + //let msb_mask = u32::from_be(0x00808080); + //let add_one = u32::from_be(0x80000000); + //((all & msb_mask) | add_one).count_ones() as usize + } + // There is no .is_emty() because this type is never empty. + + /// Checks that the codepoint is an ASCII character. + pub fn is_ascii(&self) -> bool { + self.bytes[0] <= 127 + } + /// Checks that two characters are an ASCII case-insensitive match. + /// + /// Is equivalent to `a.to_ascii_lowercase() == b.to_ascii_lowercase()`. + #[cfg(feature="std")] + pub fn eq_ignore_ascii_case(&self, other: &Self) -> bool { + if self.is_ascii() {self.bytes[0].eq_ignore_ascii_case(&other.bytes[0])} + else {self == other} + } + /// Converts the character to its ASCII upper case equivalent. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + #[cfg(feature="std")] + pub fn to_ascii_uppercase(&self) -> Self { + let mut uc = *self; + uc.make_ascii_uppercase(); + uc + } + /// Converts the character to its ASCII lower case equivalent. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + #[cfg(feature="std")] + pub fn to_ascii_lowercase(&self) -> Self { + let mut uc = *self; + uc.make_ascii_lowercase(); + uc + } + /// Converts the character to its ASCII upper case equivalent in-place. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + #[inline] + #[cfg(feature="std")] + pub fn make_ascii_uppercase(&mut self) { + self.bytes[0].make_ascii_uppercase() + } + /// Converts the character to its ASCII lower case equivalent in-place. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + #[inline] + #[cfg(feature="std")] + pub fn make_ascii_lowercase(&mut self) { + self.bytes[0].make_ascii_lowercase(); + } + + /// Convert from UTF-8 to UTF-32 + pub fn to_char(self) -> char { + self.into() + } + /// Write the internal representation to a slice, + /// and then returns the number of bytes written. + /// + /// # Panics + /// + /// Will panic the buffer is too small; + /// You can get the required length from `.len()`, + /// but a buffer of length four is always large enough. + pub fn to_slice(self, dst: &mut[u8]) -> usize { + if self.len() > dst.len() { + panic!("The provided buffer is too small."); + } + dst[..self.len()].copy_from_slice(&self.bytes[..self.len()]); + self.len() + } + /// Expose the internal array and the number of used bytes. + pub fn to_array(self) -> ([u8;4],usize) { + (self.bytes, self.len()) + } + /// Return a `str` view of the array the codepoint is stored as. + /// + /// Is an unambiguous version of `.as_ref()`. + pub fn as_str(&self) -> &str { + self.deref() + } +} diff --git a/vendor/encode_unicode/src/utf8_iterators.rs b/vendor/encode_unicode/src/utf8_iterators.rs new file mode 100644 index 0000000..891d729 --- /dev/null +++ b/vendor/encode_unicode/src/utf8_iterators.rs @@ -0,0 +1,352 @@ +/* Copyright 2016 The encode_unicode Developers + * + * Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be + * copied, modified, or distributed except according to those terms. + */ + +use utf8_char::Utf8Char; +use errors::EmptyStrError; +extern crate core; +use self::core::{mem, u32, u64}; +use self::core::ops::Not; +use self::core::fmt; +use self::core::borrow::Borrow; +#[cfg(feature="std")] +use std::io::{Read, Error as ioError}; + + + +/// Read or iterate over the bytes of the UTF-8 representation of a codepoint. +#[derive(Clone)] +pub struct Utf8Iterator (u32); + +impl From for Utf8Iterator { + fn from(uc: Utf8Char) -> Self { + let used = u32::from_le(unsafe{ mem::transmute(uc.to_array().0) }); + // uses u64 because shifting an u32 by 32 bits is a no-op. + let unused_set = (u64::MAX << uc.len() as u64*8) as u32; + Utf8Iterator(used | unused_set) + } +} +impl From for Utf8Iterator { + fn from(c: char) -> Self { + Self::from(Utf8Char::from(c)) + } +} +impl Iterator for Utf8Iterator { + type Item=u8; + fn next(&mut self) -> Option { + let next = self.0 as u8; + if next == 0xff { + None + } else { + self.0 = (self.0 >> 8) | 0xff_00_00_00; + Some(next) + } + } + fn size_hint(&self) -> (usize, Option) { + (self.len(), Some(self.len())) + } +} +impl ExactSizeIterator for Utf8Iterator { + fn len(&self) -> usize {// not straightforward, but possible + let unused_bytes = self.0.not().leading_zeros() / 8; + 4 - unused_bytes as usize + } +} +#[cfg(feature="std")] +impl Read for Utf8Iterator { + /// Always returns Ok + fn read(&mut self, buf: &mut[u8]) -> Result { + // Cannot call self.next() until I know I can write the result. + for (i, dst) in buf.iter_mut().enumerate() { + match self.next() { + Some(b) => *dst = b, + None => return Ok(i), + } + } + Ok(buf.len()) + } +} +impl fmt::Debug for Utf8Iterator { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + let mut content = [0; 4]; + let mut i = 0; + for b in self.clone() { + content[i] = b; + i += 1; + } + write!(fmtr, "{:?}", &content[..i]) + } +} + + + +/// Converts an iterator of `Utf8Char` (or `&Utf8Char`) +/// to an iterator of `u8`s. +/// Is equivalent to calling `.flat_map()` on the original iterator, +/// but the returned iterator is ~40% faster. +/// +/// The iterator also implements `Read` (if the `std` feature isn't disabled). +/// Reading will never produce an error, and calls to `.read()` and `.next()` +/// can be mixed. +/// +/// The exact number of bytes cannot be known in advance, but `size_hint()` +/// gives the possible range. +/// (min: all remaining characters are ASCII, max: all require four bytes) +/// +/// # Examples +/// +/// From iterator of values: +/// +/// ``` +/// use encode_unicode::{iter_bytes, CharExt}; +/// +/// let iterator = "foo".chars().map(|c| c.to_utf8() ); +/// let mut bytes = [0; 4]; +/// for (u,dst) in iter_bytes(iterator).zip(&mut bytes) {*dst=u;} +/// assert_eq!(&bytes, b"foo\0"); +/// ``` +/// +/// From iterator of references: +/// +#[cfg_attr(feature="std", doc=" ```")] +#[cfg_attr(not(feature="std"), doc=" ```no_compile")] +/// use encode_unicode::{iter_bytes, CharExt, Utf8Char}; +/// +/// let chars: Vec = "💣 bomb 💣".chars().map(|c| c.to_utf8() ).collect(); +/// let bytes: Vec = iter_bytes(&chars).collect(); +/// let flat_map: Vec = chars.iter().flat_map(|u8c| *u8c ).collect(); +/// assert_eq!(bytes, flat_map); +/// ``` +/// +/// `Read`ing from it: +/// +#[cfg_attr(feature="std", doc=" ```")] +#[cfg_attr(not(feature="std"), doc=" ```no_compile")] +/// use encode_unicode::{iter_bytes, CharExt}; +/// use std::io::Read; +/// +/// let s = "Ååh‽"; +/// assert_eq!(s.len(), 8); +/// let mut buf = [b'E'; 9]; +/// let mut reader = iter_bytes(s.chars().map(|c| c.to_utf8() )); +/// assert_eq!(reader.read(&mut buf[..]).unwrap(), 8); +/// assert_eq!(reader.read(&mut buf[..]).unwrap(), 0); +/// assert_eq!(&buf[..8], s.as_bytes()); +/// assert_eq!(buf[8], b'E'); +/// ``` +pub fn iter_bytes, I:IntoIterator> +(iterable: I) -> Utf8CharSplitter { + Utf8CharSplitter{ inner: iterable.into_iter(), prev: 0 } +} + +/// The iterator type returned by `iter_bytes()` +/// +/// See its documentation for details. +#[derive(Clone)] +pub struct Utf8CharSplitter, I:Iterator> { + inner: I, + prev: u32, +} +impl> From for Utf8CharSplitter { + /// A less generic constructor than `iter_bytes()` + fn from(iter: I) -> Self { + iter_bytes(iter) + } +} +impl, I:Iterator> Utf8CharSplitter { + /// Extracts the source iterator. + /// + /// Note that `iter_bytes(iter.into_inner())` is not a no-op: + /// If the last returned byte from `next()` was not an ASCII by, + /// the remaining bytes of that codepoint is lost. + pub fn into_inner(self) -> I { + self.inner + } +} +impl, I:Iterator> Iterator for Utf8CharSplitter { + type Item = u8; + fn next(&mut self) -> Option { + if self.prev == 0 { + self.inner.next().map(|u8c| { + let array = u8c.borrow().to_array().0; + self.prev = unsafe{ u32::from_le(mem::transmute(array)) } >> 8; + array[0] + }) + } else { + let next = self.prev as u8; + self.prev >>= 8; + Some(next) + } + } + fn size_hint(&self) -> (usize,Option) { + // Doesn't need to handle unlikely overflows correctly because + // size_hint() cannot be relied upon anyway. (the trait isn't unsafe) + let (min, max) = self.inner.size_hint(); + let add = 4 - (self.prev.leading_zeros() / 8) as usize; + (min.wrapping_add(add), max.map(|max| max.wrapping_mul(4).wrapping_add(add) )) + } +} +#[cfg(feature="std")] +impl, I:Iterator> Read for Utf8CharSplitter { + /// Always returns `Ok` + fn read(&mut self, buf: &mut[u8]) -> Result { + let mut i = 0; + // write remaining bytes of previous codepoint + while self.prev != 0 && i < buf.len() { + buf[i] = self.prev as u8; + self.prev >>= 8; + i += 1; + } + // write whole characters + while i < buf.len() { + let bytes = match self.inner.next() { + Some(u8c) => u8c.borrow().to_array().0, + None => break + }; + buf[i] = bytes[0]; + i += 1; + if bytes[1] != 0 { + let len = bytes[0].not().leading_zeros() as usize; + let mut written = 1; + while written < len { + if i < buf.len() { + buf[i] = bytes[written]; + i += 1; + written += 1; + } else { + let bytes_as_u32 = unsafe{ u32::from_le(mem::transmute(bytes)) }; + self.prev = bytes_as_u32 >> (8*written); + return Ok(i); + } + } + } + } + Ok(i) + } +} + + + +/// An iterator over the `Utf8Char` of a string slice, and their positions. +/// +/// This struct is created by the `utf8char_indices() method from [`StrExt`] trait. See its documentation for more. +#[derive(Clone)] +pub struct Utf8CharIndices<'a>{ + str: &'a str, + index: usize, +} +impl<'a> From<&'a str> for Utf8CharIndices<'a> { + fn from(s: &str) -> Utf8CharIndices { + Utf8CharIndices{str: s, index: 0} + } +} +impl<'a> Utf8CharIndices<'a> { + /// Extract the remainder of the source `str`. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::{StrExt, Utf8Char}; + /// let mut iter = "abc".utf8char_indices(); + /// assert_eq!(iter.next_back(), Some((2, Utf8Char::from('c')))); + /// assert_eq!(iter.next(), Some((0, Utf8Char::from('a')))); + /// assert_eq!(iter.as_str(), "b"); + /// ``` + pub fn as_str(&self) -> &'a str { + &self.str[self.index..] + } +} +impl<'a> Iterator for Utf8CharIndices<'a> { + type Item = (usize,Utf8Char); + fn next(&mut self) -> Option<(usize,Utf8Char)> { + match Utf8Char::from_str_start(&self.str[self.index..]) { + Ok((u8c, len)) => { + let item = (self.index, u8c); + self.index += len; + Some(item) + }, + Err(EmptyStrError) => None + } + } + fn size_hint(&self) -> (usize,Option) { + let len = self.str.len() - self.index; + // For len+3 to overflow, the slice must fill all but two bytes of + // addressable memory, and size_hint() doesn't need to be correct. + (len.wrapping_add(3)/4, Some(len)) + } +} +impl<'a> DoubleEndedIterator for Utf8CharIndices<'a> { + fn next_back(&mut self) -> Option<(usize,Utf8Char)> { + // Cannot refactor out the unwrap without switching to ::from_slice() + // since slicing the str panics if not on a boundary. + if self.index < self.str.len() { + let rev = self.str.bytes().rev(); + let len = 1 + rev.take_while(|b| b & 0b1100_0000 == 0b1000_0000 ).count(); + let starts = self.str.len() - len; + let (u8c,_) = Utf8Char::from_str_start(&self.str[starts..]).unwrap(); + self.str = &self.str[..starts]; + Some((starts, u8c)) + } else { + None + } + } +} +impl<'a> fmt::Debug for Utf8CharIndices<'a> { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + fmtr.debug_tuple("Utf8CharIndices") + .field(&self.index) + .field(&self.as_str()) + .finish() + } +} + + +/// An iterator over the codepoints in a `str` represented as `Utf8Char`. +#[derive(Clone)] +pub struct Utf8Chars<'a>(Utf8CharIndices<'a>); +impl<'a> From<&'a str> for Utf8Chars<'a> { + fn from(s: &str) -> Utf8Chars { + Utf8Chars(Utf8CharIndices::from(s)) + } +} +impl<'a> Utf8Chars<'a> { + /// Extract the remainder of the source `str`. + /// + /// # Examples + /// + /// ``` + /// use encode_unicode::{StrExt, Utf8Char}; + /// let mut iter = "abc".utf8chars(); + /// assert_eq!(iter.next(), Some(Utf8Char::from('a'))); + /// assert_eq!(iter.next_back(), Some(Utf8Char::from('c'))); + /// assert_eq!(iter.as_str(), "b"); + /// ``` + pub fn as_str(&self) -> &'a str { + self.0.as_str() + } +} +impl<'a> Iterator for Utf8Chars<'a> { + type Item = Utf8Char; + fn next(&mut self) -> Option { + self.0.next().map(|(_,u8c)| u8c ) + } + fn size_hint(&self) -> (usize,Option) { + self.0.size_hint() + } +} +impl<'a> DoubleEndedIterator for Utf8Chars<'a> { + fn next_back(&mut self) -> Option { + self.0.next_back().map(|(_,u8c)| u8c ) + } +} +impl<'a> fmt::Debug for Utf8Chars<'a> { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + fmtr.debug_tuple("Utf8CharIndices") + .field(&self.as_str()) + .finish() + } +} -- cgit v1.2.3