aboutsummaryrefslogtreecommitdiff
path: root/vendor/encode_unicode/src
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/encode_unicode/src')
-rw-r--r--vendor/encode_unicode/src/decoding_iterators.rs494
-rw-r--r--vendor/encode_unicode/src/errors.rs289
-rw-r--r--vendor/encode_unicode/src/lib.rs78
-rw-r--r--vendor/encode_unicode/src/traits.rs1014
-rw-r--r--vendor/encode_unicode/src/utf16_char.rs687
-rw-r--r--vendor/encode_unicode/src/utf16_iterators.rs270
-rw-r--r--vendor/encode_unicode/src/utf8_char.rs647
-rw-r--r--vendor/encode_unicode/src/utf8_iterators.rs352
8 files changed, 0 insertions, 3831 deletions
diff --git a/vendor/encode_unicode/src/decoding_iterators.rs b/vendor/encode_unicode/src/decoding_iterators.rs
deleted file mode 100644
index 4ef4125..0000000
--- a/vendor/encode_unicode/src/decoding_iterators.rs
+++ /dev/null
@@ -1,494 +0,0 @@
-/* Copyright 2018 The encode_unicode Developers
- *
- * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
- * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
- * http://opensource.org/licenses/MIT>, at your option. This file may not be
- * copied, modified, or distributed except according to those terms.
- */
-
-//! Iterators that turn multiple `u8`s or `u16`s into `Utf*Char`s, but can fail.
-//!
-//! To be predictable, all errors consume one element each.
-//!
-//! The iterator adaptors produce neither offset nor element length to work
-//! well with other adaptors,
-//! while the slice iterators yield both to make more advanced use cases easy.
-
-use errors::{InvalidUtf8Slice, InvalidUtf16FirstUnit, Utf16PairError};
-use errors::InvalidUtf8Slice::*;
-use errors::InvalidUtf8::*;
-use errors::InvalidUtf8FirstByte::*;
-use errors::InvalidUtf16Slice::*;
-use errors::InvalidCodepoint::*;
-use errors::Utf16PairError::*;
-use utf8_char::Utf8Char;
-use utf16_char::Utf16Char;
-use traits::U16UtfExt;
-extern crate core;
-use self::core::borrow::Borrow;
-use self::core::fmt::{self, Debug};
-use self::core::iter::Chain;
-use self::core::option;
-
-
-/// Decodes UTF-8 characters from a byte iterator into `Utf8Char`s.
-///
-/// See [`IterExt::to_utf8chars()`](../trait.IterExt.html#tymethod.to_utf8chars)
-/// for examples and error handling.
-#[derive(Clone, Default)]
-pub struct Utf8CharMerger<B:Borrow<u8>, I:Iterator<Item=B>> {
- iter: I,
- /// number of bytes that were read before an error was detected
- after_err_leftover: u8,
- /// stack because it simplifies popping.
- after_err_stack: [u8; 3],
-}
-impl<B:Borrow<u8>, I:Iterator<Item=B>, T:IntoIterator<IntoIter=I,Item=B>>
-From<T> for Utf8CharMerger<B, I> {
- fn from(t: T) -> Self {
- Utf8CharMerger {
- iter: t.into_iter(),
- after_err_leftover: 0,
- after_err_stack: [0; 3],
- }
- }
-}
-impl<B:Borrow<u8>, I:Iterator<Item=B>> Utf8CharMerger<B,I> {
- /// Extract the inner iterator.
- ///
- /// If the last item produced by `.next()` was an `Err`,
- /// up to three following bytes might be missing.
- /// The exact number of missing bytes for each error type should not be relied on.
- ///
- /// # Examples
- ///
- /// Three bytes swallowed:
- /// ```
- /// # use encode_unicode::IterExt;
- /// let mut merger = b"\xf4\xa1\xb2FS".iter().to_utf8chars();
- /// assert!(merger.next().unwrap().is_err());
- /// let mut inner: std::slice::Iter<u8> = merger.into_inner();
- /// assert_eq!(inner.next(), Some(&b'S')); // b'\xa1', b'\xb2' and b'F' disappeared
- /// ```
- ///
- /// All bytes present:
- /// ```
- /// # use encode_unicode::IterExt;
- /// let mut merger = b"\xb0FS".iter().to_utf8chars();
- /// assert!(merger.next().unwrap().is_err());
- /// assert_eq!(merger.into_inner().next(), Some(&b'F'));
- /// ```
- ///
- /// Two bytes missing:
- /// ```
- /// # use encode_unicode::IterExt;
- /// let mut merger = b"\xe0\x80\x80FS".iter().to_utf8chars();
- /// assert!(merger.next().unwrap().is_err());
- /// assert_eq!(merger.into_inner().next(), Some(&b'F'));
- /// ```
- pub fn into_inner(self) -> I {
- self.iter
- }
-
- fn save(&mut self, bytes: &[u8;4], len: usize) {
- // forget bytes[0] and push the others onto self.after_err_stack (in reverse).
- for &after_err in bytes[1..len].iter().rev() {
- self.after_err_stack[self.after_err_leftover as usize] = after_err;
- self.after_err_leftover += 1;
- }
- }
- /// Reads len-1 bytes into bytes[1..]
- fn extra(&mut self, bytes: &mut[u8;4], len: usize) -> Result<(),InvalidUtf8Slice> {
- // This is the only function that pushes onto after_err_stack,
- // and it checks that all bytes are continuation bytes before fetching the next one.
- // Therefore only the last byte retrieved can be a non-continuation byte.
- // That last byte is also the last to be retrieved from after_err.
- //
- // Before this function is called, there has been retrieved at least one byte.
- // If that byte was a continuation byte, next() produces an error
- // and won't call this function.
- // Therefore, we know that after_err is empty at this point.
- // This means that we can use self.iter directly, and knows where to start pushing
- debug_assert_eq!(self.after_err_leftover, 0, "first: {:#02x}, stack: {:?}", bytes[0], self.after_err_stack);
- for i in 1..len {
- if let Some(extra) = self.iter.next() {
- let extra = *extra.borrow();
- bytes[i] = extra;
- if extra & 0b1100_0000 != 0b1000_0000 {
- // not a continuation byte
- self.save(bytes, i+1);
- return Err(InvalidUtf8Slice::Utf8(NotAContinuationByte(i)))
- }
- } else {
- self.save(bytes, i);
- return Err(TooShort(len));
- }
- }
- Ok(())
- }
-}
-impl<B:Borrow<u8>, I:Iterator<Item=B>> Iterator for Utf8CharMerger<B,I> {
- type Item = Result<Utf8Char,InvalidUtf8Slice>;
- fn next(&mut self) -> Option<Self::Item> {
- let first: u8;
- if self.after_err_leftover != 0 {
- self.after_err_leftover -= 1;
- first = self.after_err_stack[self.after_err_leftover as usize];
- } else if let Some(next) = self.iter.next() {
- first = *next.borrow();
- } else {
- return None;
- }
-
- unsafe {
- let mut bytes = [first, 0, 0, 0];
- let ok = match first {
- 0b0000_0000...0b0111_1111 => {/*1 and */Ok(())},
- 0b1100_0010...0b1101_1111 => {//2 and not overlong
- self.extra(&mut bytes, 2) // no extra validation required
- },
- 0b1110_0000...0b1110_1111 => {//3
- if let Err(e) = self.extra(&mut bytes, 3) {
- Err(e)
- } else if bytes[0] == 0b1110_0000 && bytes[1] <= 0b10_011111 {
- self.save(&bytes, 3);
- Err(Utf8(OverLong))
- } else if bytes[0] == 0b1110_1101 && bytes[1] & 0b11_100000 == 0b10_100000 {
- self.save(&bytes, 3);
- Err(Codepoint(Utf16Reserved))
- } else {
- Ok(())
- }
- },
- 0b1111_0000...0b1111_0100 => {//4
- if let Err(e) = self.extra(&mut bytes, 4) {
- Err(e)
- } else if bytes[0] == 0b11110_000 && bytes[1] <= 0b10_001111 {
- self.save(&bytes, 4);
- Err(InvalidUtf8Slice::Utf8(OverLong))
- } else if bytes[0] == 0b11110_100 && bytes[1] > 0b10_001111 {
- self.save(&bytes, 4);
- Err(InvalidUtf8Slice::Codepoint(TooHigh))
- } else {
- Ok(())
- }
- },
- 0b1000_0000...0b1011_1111 => {// continuation byte
- Err(Utf8(FirstByte(ContinuationByte)))
- },
- 0b1100_0000...0b1100_0001 => {// 2 and overlong
- Err(Utf8(OverLong))
- },
- 0b1111_0101...0b1111_0111 => {// 4 and too high codepoint
- Err(Codepoint(TooHigh))
- },
- 0b1111_1000...0b1111_1111 => {
- Err(Utf8(FirstByte(TooLongSeqence)))
- },
- _ => unreachable!("all possible byte values should be covered")
- };
- Some(ok.map(|()| Utf8Char::from_array_unchecked(bytes) ))
- }
- }
- fn size_hint(&self) -> (usize,Option<usize>) {
- let (iter_min, iter_max) = self.iter.size_hint();
- // cannot be exact, so KISS
- let min = iter_min / 4; // don't bother rounding up or accounting for after_err
- // handle edge case of max > usize::MAX-3 just in case.
- // Using wrapping_add() wouldn't violate any API contract as the trait isn't unsafe.
- let max = iter_max.and_then(|max| {
- max.checked_add(self.after_err_leftover as usize)
- });
- (min, max)
- }
-}
-impl<B:Borrow<u8>, I:Iterator<Item=B>+Debug> Debug for Utf8CharMerger<B,I> {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- let mut in_order = [0u8; 3];
- for i in 0..self.after_err_leftover as usize {
- in_order[i] = self.after_err_stack[self.after_err_leftover as usize - i - 1];
- }
- fmtr.debug_struct("Utf8CharMerger")
- .field("buffered", &&in_order[..self.after_err_leftover as usize])
- .field("inner", &self.iter)
- .finish()
- }
-}
-
-
-/// An [`Utf8CharMerger`](struct.Utf8CharMerger.html) that also produces
-/// offsets and lengths, but can only iterate over slices.
-///
-/// See [`SliceExt::utf8char_indices()`](../trait.SliceExt.html#tymethod.utf8char_indices)
-/// for examples and error handling.
-#[derive(Clone, Default)]
-pub struct Utf8CharDecoder<'a> {
- slice: &'a[u8],
- index: usize,
-}
-impl<'a> From<&'a[u8]> for Utf8CharDecoder<'a> {
- fn from(s: &[u8]) -> Utf8CharDecoder {
- Utf8CharDecoder { slice: s, index: 0 }
- }
-}
-impl<'a> Utf8CharDecoder<'a> {
- /// Extract the remainder of the source slice.
- ///
- /// # Examples
- ///
- /// Unlike `Utf8CharMerger::into_inner()`, bytes directly after an error
- /// are never swallowed:
- /// ```
- /// # use encode_unicode::SliceExt;
- /// let mut iter = b"\xf4\xa1\xb2FS".utf8char_indices();
- /// assert!(iter.next().unwrap().1.is_err());
- /// assert_eq!(iter.as_slice(), b"\xa1\xb2FS");
- /// ```
- pub fn as_slice(&self) -> &'a[u8] {
- &self.slice[self.index..]
- }
-}
-impl<'a> Iterator for Utf8CharDecoder<'a> {
- type Item = (usize, Result<Utf8Char,InvalidUtf8Slice>, usize);
- fn next(&mut self) -> Option<Self::Item> {
- let start = self.index;
- match Utf8Char::from_slice_start(&self.slice[self.index..]) {
- Ok((u8c, len)) => {
- self.index += len;
- Some((start, Ok(u8c), len))
- },
- Err(TooShort(1)) => None,
- Err(e) => {
- self.index += 1;
- Some((start, Err(e), 1))
- }
- }
- }
- #[inline]
- fn size_hint(&self) -> (usize,Option<usize>) {
- let bytes = self.slice.len() - self.index;
- // Cannot be exact, so KISS and don't bother rounding up.
- // The slice is unlikely be full of 4-byte codepoints, so buffers
- // allocated with the lower bound will have to be grown anyway.
- (bytes/4, Some(bytes))
- }
-}
-impl<'a> DoubleEndedIterator for Utf8CharDecoder<'a> {
- fn next_back(&mut self) -> Option<Self::Item> {
- if self.index < self.slice.len() {
- let extras = self.slice.iter()
- .rev()
- .take_while(|&b| b & 0b1100_0000 == 0b1000_0000 )
- .count();
- let starts = self.slice.len() - (extras+1);
- match Utf8Char::from_slice_start(&self.slice[starts..]) {
- Ok((u8c,len)) if len == 1+extras => {
- self.slice = &self.slice[..starts];
- Some((starts, Ok(u8c), len))
- },
- // This enures errors for every byte in both directions,
- // but means overlong and codepoint errors will be turned into
- // tooshort errors.
- Err(e) if extras == 0 => {
- self.slice = &self.slice[..self.slice.len()-1];
- Some((self.slice.len()-1, Err(e), 1))
- },
- _ => {
- self.slice = &self.slice[..self.slice.len()-1];
- Some((self.slice.len()-1, Err(Utf8(FirstByte(ContinuationByte))), 1))
- },
- }
- } else {
- None
- }
- }
-}
-impl<'a> Debug for Utf8CharDecoder<'a> {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- write!(fmtr, "Utf8CharDecoder {{ bytes[{}..]: {:?} }}", self.index, self.as_slice())
- }
-}
-
-
-
-/// Decodes UTF-16 characters from a `u16` iterator into `Utf16Char`s.
-///
-/// See [`IterExt::to_utf16chars()`](../trait.IterExt.html#tymethod.to_utf16chars)
-/// for examples and error handling.
-#[derive(Clone, Default)]
-pub struct Utf16CharMerger<B:Borrow<u16>, I:Iterator<Item=B>> {
- iter: I,
- /// Used when a trailing surrogate was expected, the u16 can be any value.
- prev: Option<B>,
-}
-impl<B:Borrow<u16>, I:Iterator<Item=B>, T:IntoIterator<IntoIter=I,Item=B>>
-From<T> for Utf16CharMerger<B,I> {
- fn from(t: T) -> Self {
- Utf16CharMerger { iter: t.into_iter(), prev: None }
- }
-}
-impl<B:Borrow<u16>, I:Iterator<Item=B>> Utf16CharMerger<B,I> {
- /// Extract the inner iterator.
- ///
- /// If the last item produced was an `Err`, the first unit might be missing.
- ///
- /// # Examples
- ///
- /// Unit right after an error missing
- /// ```
- /// # use encode_unicode::IterExt;
- /// # use encode_unicode::error::Utf16PairError;
- /// let mut merger = [0xd901, 'F' as u16, 'S' as u16].iter().to_utf16chars();
- /// assert_eq!(merger.next(), Some(Err(Utf16PairError::UnmatchedLeadingSurrogate)));
- /// let mut inner: std::slice::Iter<u16> = merger.into_inner();
- /// assert_eq!(inner.next(), Some('S' as u16).as_ref()); // 'F' was consumed by Utf16CharMerger
- /// ```
- ///
- /// Error that doesn't swallow any units
- /// ```
- /// # use encode_unicode::IterExt;
- /// # use encode_unicode::error::Utf16PairError;
- /// let mut merger = [0xde00, 'F' as u16, 'S' as u16].iter().to_utf16chars();
- /// assert_eq!(merger.next(), Some(Err(Utf16PairError::UnexpectedTrailingSurrogate)));
- /// let mut inner: std::slice::Iter<u16> = merger.into_inner();
- /// assert_eq!(inner.next(), Some('F' as u16).as_ref()); // not consumed
- /// ```
- pub fn into_inner(self) -> I {
- self.iter
- }
- /// Returns an iterator over the remaining units.
- /// Unlike `into_inner()` this will never drop any units.
- ///
- /// The exact type of the returned iterator should not be depended on.
- ///
- /// # Examples
- ///
- /// ```
- /// # use encode_unicode::IterExt;
- /// # use encode_unicode::error::Utf16PairError;
- /// let slice = [0xd901, 'F' as u16, 'S' as u16];
- /// let mut merger = slice.iter().to_utf16chars();
- /// assert_eq!(merger.next(), Some(Err(Utf16PairError::UnmatchedLeadingSurrogate)));
- /// let mut remaining = merger.into_remaining_units();
- /// assert_eq!(remaining.next(), Some('F' as u16).as_ref());
- /// ```
- pub fn into_remaining_units(self) -> Chain<option::IntoIter<B>,I> {
- self.prev.into_iter().chain(self.iter)
- }
-}
-impl<B:Borrow<u16>, I:Iterator<Item=B>> Iterator for Utf16CharMerger<B,I> {
- type Item = Result<Utf16Char,Utf16PairError>;
- fn next(&mut self) -> Option<Self::Item> {
- let first = self.prev.take().or_else(|| self.iter.next() );
- first.map(|first| unsafe {
- match first.borrow().utf16_needs_extra_unit() {
- Ok(false) => Ok(Utf16Char::from_array_unchecked([*first.borrow(), 0])),
- Ok(true) => match self.iter.next() {
- Some(second) => match second.borrow().utf16_needs_extra_unit() {
- Err(InvalidUtf16FirstUnit) => Ok(Utf16Char::from_tuple_unchecked((
- *first.borrow(),
- Some(*second.borrow())
- ))),
- Ok(_) => {
- self.prev = Some(second);
- Err(Utf16PairError::UnmatchedLeadingSurrogate)
- }
- },
- None => Err(Utf16PairError::Incomplete)
- },
- Err(InvalidUtf16FirstUnit) => Err(Utf16PairError::UnexpectedTrailingSurrogate),
- }
- })
- }
- fn size_hint(&self) -> (usize,Option<usize>) {
- let (iter_min, iter_max) = self.iter.size_hint();
- // cannot be exact, so KISS
- let min = iter_min / 2; // don't bother rounding up or accounting for self.prev
- let max = match (iter_max, &self.prev) {
- (Some(max), &Some(_)) => max.checked_add(1),
- (max, _) => max,
- };
- (min, max)
- }
-}
-impl<B:Borrow<u16>, I:Iterator<Item=B>+Debug> Debug for Utf16CharMerger<B,I> {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- fmtr.debug_struct("Utf16CharMerger")
- .field("buffered", &self.prev.as_ref().map(|b| *b.borrow() ))
- .field("inner", &self.iter)
- .finish()
- }
-}
-
-
-/// An [`Utf16CharMerger`](struct.Utf16CharMerger.html) that also produces
-/// offsets and lengths, but can only iterate over slices.
-///
-/// See [`SliceExt::utf16char_indices()`](../trait.SliceExt.html#tymethod.utf16char_indices)
-/// for examples and error handling.
-#[derive(Clone, Default)]
-pub struct Utf16CharDecoder<'a> {
- slice: &'a[u16],
- index: usize,
-}
-impl<'a> From<&'a[u16]> for Utf16CharDecoder<'a> {
- fn from(s: &'a[u16]) -> Self {
- Utf16CharDecoder{ slice: s, index: 0 }
- }
-}
-impl<'a> Utf16CharDecoder<'a> {
- /// Extract the remainder of the source slice.
- ///
- /// # Examples
- ///
- /// Unlike `Utf16CharMerger::into_inner()`, the unit after an error is never swallowed:
- /// ```
- /// # use encode_unicode::SliceExt;
- /// # use encode_unicode::error::Utf16PairError;
- /// let mut iter = [0xd901, 'F' as u16, 'S' as u16].utf16char_indices();
- /// assert_eq!(iter.next(), Some((0, Err(Utf16PairError::UnmatchedLeadingSurrogate), 1)));
- /// assert_eq!(iter.as_slice(), &['F' as u16, 'S' as u16]);
- /// ```
- pub fn as_slice(&self) -> &[u16] {
- &self.slice[self.index..]
- }
-}
-impl<'a> Iterator for Utf16CharDecoder<'a> {
- type Item = (usize,Result<Utf16Char,Utf16PairError>,usize);
- #[inline]
- fn next(&mut self) -> Option<Self::Item> {
- let start = self.index;
- match Utf16Char::from_slice_start(self.as_slice()) {
- Ok((u16c,len)) => {
- self.index += len;
- Some((start, Ok(u16c), len))
- },
- Err(EmptySlice) => None,
- Err(FirstLowSurrogate) => {
- self.index += 1;
- Some((start, Err(UnexpectedTrailingSurrogate), 1))
- },
- Err(SecondNotLowSurrogate) => {
- self.index += 1;
- Some((start, Err(UnmatchedLeadingSurrogate), 1))
- },
- Err(MissingSecond) => {
- self.index = self.slice.len();
- Some((start, Err(Incomplete), 1))
- }
- }
- }
- #[inline]
- fn size_hint(&self) -> (usize,Option<usize>) {
- let units = self.slice.len() - self.index;
- // Cannot be exact, so KISS and don't bother rounding up.
- // The slice is unlikely be full of surrogate pairs, so buffers
- // allocated with the lower bound will have to be grown anyway.
- (units/2, Some(units))
- }
-}
-impl<'a> Debug for Utf16CharDecoder<'a> {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- write!(fmtr, "Utf16CharDecoder {{ units[{}..]: {:?} }}", self.index, self.as_slice())
- }
-}
diff --git a/vendor/encode_unicode/src/errors.rs b/vendor/encode_unicode/src/errors.rs
deleted file mode 100644
index fb587fa..0000000
--- a/vendor/encode_unicode/src/errors.rs
+++ /dev/null
@@ -1,289 +0,0 @@
-/* Copyright 2016 The encode_unicode Developers
- *
- * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
- * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
- * http://opensource.org/licenses/MIT>, at your option. This file may not be
- * copied, modified, or distributed except according to those terms.
- */
-
-
-//! Boilerplatey error types
-
-extern crate core;
-use self::core::fmt::{self,Display,Formatter};
-#[cfg(feature="std")]
-use std::error::Error;
-
-
-macro_rules! description {($err:ty, $desc:expr) => {
- #[cfg(not(feature="std"))]
- impl $err {
- #[allow(missing_docs)]
- pub fn description(&self) -> &'static str {
- ($desc)(self)
- }
- }
- #[cfg(feature="std")]
- impl Error for $err {
- fn description(&self) -> &'static str {
- ($desc)(self)
- }
- }
- impl Display for $err {
- fn fmt(&self, fmtr: &mut Formatter) -> fmt::Result {
- write!(fmtr, "{}", self.description())
- }
- }
-}}
-
-
-macro_rules! single_cause {(#[$doc1:meta] #[$doc2:meta] $err:ident => $desc:expr) => {
- // Rust 1.15 doesn't understand $(#[$doc:meta])* $:ident
- #[$doc1]
- #[$doc2]
- #[derive(Clone,Copy, Debug, PartialEq,Eq)]
- pub struct $err;
- description!{$err, |_| $desc }
-}}
-
-
-single_cause!{
- /// Cannot tell whether an `u16` needs an extra unit,
- /// because it's a trailing surrogate itself.
- InvalidUtf16FirstUnit => "is a trailing surrogate"
-}
-
-single_cause!{
- /// Cannot create an `Utf8Char` or `Utf16Char` from the first codepoint of a str,
- /// because there are none.
- EmptyStrError => "is empty"
-}
-
-single_cause!{
- /// Cannot create an `Utf8Char` from a standalone `u8`
- /// that is not an ASCII character.
- NonAsciiError => "is not an ASCII character"
-}
-
-single_cause!{
- /// Cannot create an `Utf16Char` from a standalone `u16` that is not a
- /// codepoint in the basic multilingual plane, but part of a suurrogate pair.
- NonBMPError => "is not a codepoint in the basic multilingual plane"
-}
-
-
-
-macro_rules! simple {(#[$tydoc:meta] $err:ident {
- $($(#[$vardoc:meta])* ::$variant:ident => $string:expr),+,
- } ) => {
- #[$tydoc]
- #[derive(Clone,Copy, Debug, PartialEq,Eq)]
- pub enum $err {
- $($(#[$vardoc])* $variant),*
- }
- description!{$err, |e: &$err| match *e {$($err::$variant=>$string),*} }
-}}
-
-
-simple!{/// Reasons why an `u32` is not a valid UTF codepoint.
- InvalidCodepoint {
- /// It's reserved for UTF-16 surrogate pairs."
- ::Utf16Reserved => "is reserved for UTF-16 surrogate pairs",
- /// It's higher than the highest codepoint (which is 0x10ffff).
- ::TooHigh => "is higher than the highest codepoint",
- }}
-use self::InvalidCodepoint::*;
-impl InvalidCodepoint {
- /// Get the range of values for which this error would be given.
- pub fn error_range(self) -> (u32,u32) {match self {
- Utf16Reserved => (0xd8_00, 0xdf_ff),
- TooHigh => (0x00_10_ff_ff, 0xff_ff_ff_ff),
- }}
-}
-
-
-simple!{/// Reasons why a `[u16; 2]` doesn't form a valid UTF-16 codepoint.
- InvalidUtf16Array {
- /// The first unit is a trailing/low surrogate, which is never valid.
- ::FirstIsTrailingSurrogate => "the first unit is a trailing surrogate, which is never valid",
- /// The second unit is needed, but is not a trailing surrogate.
- ::SecondIsNotTrailingSurrogate => "the second unit is needed but is not a trailing surrogate",
- }}
-
-simple!{/// Reasons why one or two `u16`s are not valid UTF-16, in sinking precedence.
- InvalidUtf16Tuple {
- /// The first unit is a trailing/low surrogate, which is never valid.
- ///
- /// Note that the value of a low surrogate is actually higher than a high surrogate.
- ::FirstIsTrailingSurrogate => "the first unit is a trailing / low surrogate, which is never valid",
- /// You provided a second unit, but the first one stands on its own.
- ::SuperfluousSecond => "the second unit is superfluous",
- /// The first and only unit requires a second unit.
- ::MissingSecond => "the first unit requires a second unit",
- /// The first unit requires a second unit, but it's not a trailing/low surrogate.
- ///
- /// Note that the value of a low surrogate is actually higher than a high surrogate.
- ::InvalidSecond => "the required second unit is not a trailing / low surrogate",
- }}
-
-
-simple!{/// Reasons why a slice of `u16`s doesn't start with valid UTF-16.
- InvalidUtf16Slice {
- /// The slice is empty.
- ::EmptySlice => "the slice is empty",
- /// The first unit is a low surrogate.
- ::FirstLowSurrogate => "the first unit is a trailing surrogate",
- /// The first and only unit requires a second unit.
- ::MissingSecond => "the first and only unit requires a second one",
- /// The first unit requires a second one, but it's not a trailing surrogate.
- ::SecondNotLowSurrogate => "the required second unit is not a trailing surrogate",
- }}
-
-simple!{/// Types of invalid sequences encountered by `Utf16CharParser`.
- Utf16PairError {
- /// A trailing surrogate was not preceeded by a leading surrogate.
- ::UnexpectedTrailingSurrogate => "a trailing surrogate was not preceeded by a leading surrogate",
- /// A leading surrogate was followed by an unit that was not a trailing surrogate.
- ::UnmatchedLeadingSurrogate => "a leading surrogate was followed by an unit that was not a trailing surrogate",
- /// A trailing surrogate was expected when the end was reached.
- ::Incomplete => "a trailing surrogate was expected when the end was reached",
- }}
-
-
-simple!{/// Reasons why `Utf8Char::from_str()` or `Utf16Char::from_str()` failed.
- FromStrError {
- /// `Utf8Char` or `Utf16Char` cannot store more than a single codepoint.
- ::MultipleCodepoints => "has more than one codepoint",
- /// `Utf8Char` or `Utf16Char` cannot be empty.
- ::Empty => "is empty",
- }}
-
-
-simple!{/// Reasons why a byte is not the start of a UTF-8 codepoint.
- InvalidUtf8FirstByte {
- /// Sequences cannot be longer than 4 bytes. Is given for values >= 240.
- ::TooLongSeqence => "is greater than 247 (UTF-8 sequences cannot be longer than four bytes)",
- /// This byte belongs to a previous sequence. Is given for values between 128 and 192 (exclusive).
- ::ContinuationByte => "is a continuation of a previous sequence",
- }}
-use self::InvalidUtf8FirstByte::*;
-
-
-
-macro_rules! complex {
-($err:ty
- {$($sub:ty => $to:expr,)*}
- {$($desc:pat => $string:expr),+,}
- => $use_cause:expr =>
- {$($cause:pat => $result:expr),+,} $(#[$causedoc:meta])*
-) => {
- $(impl From<$sub> for $err {
- fn from(error: $sub) -> $err {
- $to(error)
- }
- })*
- #[cfg(not(feature="std"))]
- impl $err {
- #[allow(missing_docs)]
- pub fn description(&self) -> &'static str {
- match *self{ $($desc => $string,)* }
- }
- /// A hack to avoid two Display impls
- fn cause(&self) -> Option<&Display> {None}
- }
- #[cfg(feature="std")]
- impl Error for $err {
- fn description(&self) -> &'static str {
- match *self{ $($desc => $string,)* }
- }
- $(#[$causedoc])*
- fn cause(&self) -> Option<&Error> {
- match *self{ $($cause => $result,)* }
- }
- }
- impl Display for $err {
- fn fmt(&self, fmtr: &mut Formatter) -> fmt::Result {
- match (self.cause(), $use_cause) {
- (Some(d),true) => write!(fmtr, "{}: {}", self.description(), d),
- _ => write!(fmtr, "{}", self.description()),
- }
- }
- }
-}}
-
-
-/// Reasons why a byte sequence is not valid UTF-8, excluding invalid codepoint.
-/// In sinking precedence.
-#[derive(Clone,Copy, Debug, PartialEq,Eq)]
-pub enum InvalidUtf8 {
- /// Something is wrong with the first byte.
- FirstByte(InvalidUtf8FirstByte),
- /// The byte at index 1...3 should be a continuation byte,
- /// but dosesn't fit the pattern 0b10xx_xxxx.
- NotAContinuationByte(usize),
- /// There are too many leading zeros: it could be a byte shorter.
- ///
- /// [Decoding this could allow someone to input otherwise prohibited
- /// characters and sequences, such as "../"](https://tools.ietf.org/html/rfc3629#section-10).
- OverLong,
-}
-use self::InvalidUtf8::*;
-complex!{InvalidUtf8 {
- InvalidUtf8FirstByte => FirstByte,
- } {
- FirstByte(TooLongSeqence) => "the first byte is greater than 239 (UTF-8 sequences cannot be longer than four bytes)",
- FirstByte(ContinuationByte) => "the first byte is a continuation of a previous sequence",
- OverLong => "the sequence contains too many zeros and could be shorter",
- NotAContinuationByte(_) => "the sequence is too short",
- } => false => {
- FirstByte(ref cause) => Some(cause),
- _ => None,
- }/// Returns `Some` if the error is a `InvalidUtf8FirstByte`.
-}
-
-
-/// Reasons why a byte array is not valid UTF-8, in sinking precedence.
-#[derive(Clone,Copy, Debug, PartialEq,Eq)]
-pub enum InvalidUtf8Array {
- /// Not a valid UTF-8 sequence.
- Utf8(InvalidUtf8),
- /// Not a valid unicode codepoint.
- Codepoint(InvalidCodepoint),
-}
-complex!{InvalidUtf8Array {
- InvalidUtf8 => InvalidUtf8Array::Utf8,
- InvalidCodepoint => InvalidUtf8Array::Codepoint,
- } {
- InvalidUtf8Array::Utf8(_) => "the sequence is invalid UTF-8",
- InvalidUtf8Array::Codepoint(_) => "the encoded codepoint is invalid",
- } => true => {
- InvalidUtf8Array::Utf8(ref u) => Some(u),
- InvalidUtf8Array::Codepoint(ref c) => Some(c),
- }/// Always returns `Some`.
-}
-
-
-/// Reasons why a byte slice is not valid UTF-8, in sinking precedence.
-#[derive(Clone,Copy, Debug, PartialEq,Eq)]
-pub enum InvalidUtf8Slice {
- /// Something is certainly wrong with the first byte.
- Utf8(InvalidUtf8),
- /// The encoded codepoint is invalid:
- Codepoint(InvalidCodepoint),
- /// The slice is too short; n bytes was required.
- TooShort(usize),
-}
-complex!{InvalidUtf8Slice {
- InvalidUtf8 => InvalidUtf8Slice::Utf8,
- InvalidCodepoint => InvalidUtf8Slice::Codepoint,
- } {
- InvalidUtf8Slice::Utf8(_) => "the sequence is invalid UTF-8",
- InvalidUtf8Slice::Codepoint(_) => "the encoded codepoint is invalid",
- InvalidUtf8Slice::TooShort(1) => "the slice is empty",
- InvalidUtf8Slice::TooShort(_) => "the slice is shorter than the sequence",
- } => true => {
- InvalidUtf8Slice::Utf8(ref u) => Some(u),
- InvalidUtf8Slice::Codepoint(ref c) => Some(c),
- InvalidUtf8Slice::TooShort(_) => None,
- }
-}
diff --git a/vendor/encode_unicode/src/lib.rs b/vendor/encode_unicode/src/lib.rs
deleted file mode 100644
index c50e126..0000000
--- a/vendor/encode_unicode/src/lib.rs
+++ /dev/null
@@ -1,78 +0,0 @@
-/* Copyright 2016 The encode_unicode Developers
- *
- * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
- * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
- * http://opensource.org/licenses/MIT>, at your option. This file may not be
- * copied, modified, or distributed except according to those terms.
- */
-
-
-/*!
-Miscellaneous UTF-8 and UTF-16 types and methods.
-
-# Optional features:
-* `#![no_std]`-mode: There are a few differences:
- * `Error` doesn't exist, but `description()` is made available as an inherent impl.
- * `Extend`/`FromIterator`-implementations for `String`/`Vec<u8>`/`Vec<u16>` are missing.
- * There is no `io`, so `Utf8Iterator` and `Utf8CharSplitter` doesn't implement `Read`.
-
- This feature is enabled by setting `default-features=false` in `Cargo.toml`:
- `encode_unicode = {version="0.3.4", default-features=false}`
-* Integration with the [ascii](https://tomprogrammer.github.io/rust-ascii/ascii/index.html) crate:
- Convert `Utf8Char` and `Utf16Char` to and from
- [ascii::`AsciiChar`](https://tomprogrammer.github.io/rust-ascii/ascii/enum.AsciiChar.html).
-
-The minimum supported version of Rust is 1.15,
-older versions might work now but can break with a minor update.
-
-[crates.io page](https://crates.io/crates/encode_unicode)
-[github repository](https://github.com/tormol/encode_unicode)
-
-*/
-
-#![warn(missing_docs)]
-
-#![cfg_attr(not(feature="std"), no_std)]
-// either `cargo clippy` doesn't see theese, or I get a warning when I build.
-#![cfg_attr(feature="clippy", feature(plugin))]
-#![cfg_attr(feature="clippy", plugin(clippy))]
-#![cfg_attr(feature="clippy", allow(derive_hash_xor_eq))]// tested
-#![cfg_attr(feature="clippy", allow(len_without_is_empty))]// UtfxChar is never empty
-#![cfg_attr(feature="clippy", allow(match_same_arms))]// looks better IMO
-#![cfg_attr(feature="clippy", allow(needless_return))]// `foo.bar(); foo` looks unfinished
-#![cfg_attr(feature="clippy", allow(redundant_closure))]// keep it explicit
-#![cfg_attr(feature="clippy", allow(redundant_closure_call))]// not redundant in macros
-#![cfg_attr(feature="clippy", allow(cast_lossless))]// too much noise (and too verbose)
-// precedence: I prefer spaces to parentheses, but it's nice to recheck.
-
-mod errors;
-mod traits;
-mod utf8_char;
-mod utf8_iterators;
-mod utf16_char;
-mod utf16_iterators;
-mod decoding_iterators;
-
-pub use traits::{CharExt, U8UtfExt, U16UtfExt, StrExt, IterExt, SliceExt};
-pub use utf8_char::Utf8Char;
-pub use utf16_char::Utf16Char;
-pub use utf8_iterators::{Utf8Iterator, iter_bytes};
-pub use utf16_iterators::{Utf16Iterator, iter_units};
-
-pub mod error {// keeping the public interface in one file
- //! Errors returned by various conversion methods in this crate.
- pub use errors::{FromStrError, EmptyStrError};
- pub use errors::{InvalidCodepoint, InvalidUtf8};
- pub use errors::{InvalidUtf8FirstByte,InvalidUtf16FirstUnit};
- pub use errors::{InvalidUtf8Slice,InvalidUtf16Slice};
- pub use errors::{InvalidUtf8Array,InvalidUtf16Array,InvalidUtf16Tuple};
- pub use errors::Utf16PairError;
-}
-
-pub mod iterator {
- //! Iterator types that you should rarely need to name
- pub use utf8_iterators::{Utf8Iterator, Utf8CharSplitter, Utf8Chars, Utf8CharIndices};
- pub use utf16_iterators::{Utf16Iterator, Utf16CharSplitter, Utf16Chars, Utf16CharIndices};
- pub use decoding_iterators::{Utf8CharMerger, Utf8CharDecoder};
- pub use decoding_iterators::{Utf16CharMerger, Utf16CharDecoder};
-}
diff --git a/vendor/encode_unicode/src/traits.rs b/vendor/encode_unicode/src/traits.rs
deleted file mode 100644
index 3f36903..0000000
--- a/vendor/encode_unicode/src/traits.rs
+++ /dev/null
@@ -1,1014 +0,0 @@
-/* Copyright 2016 The encode_unicode Developers
- *
- * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
- * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
- * http://opensource.org/licenses/MIT>, at your option. This file may not be
- * copied, modified, or distributed except according to those terms.
- */
-
-#![allow(unused_unsafe)]// explicit unsafe{} blocks in unsafe functions are a good thing.
-
-use utf8_char::Utf8Char;
-use utf16_char::Utf16Char;
-use utf8_iterators::*;
-use utf16_iterators::*;
-use decoding_iterators::*;
-use error::*;
-extern crate core;
-use self::core::{char, u32, mem};
-use self::core::ops::{Not, Index, RangeFull};
-use self::core::borrow::Borrow;
-#[cfg(feature="ascii")]
-extern crate ascii;
-#[cfg(feature="ascii")]
-use self::ascii::AsciiStr;
-
-// TODO better docs and tests
-
-/// Methods for working with `u8`s as UTF-8 bytes.
-pub trait U8UtfExt {
- /// How many more bytes will you need to complete this codepoint?
- ///
- /// # Errors
- ///
- /// An error is returned if the byte is not a valid start of an UTF-8
- /// codepoint:
- ///
- /// * `128..192`: ContinuationByte
- /// * `248..`: TooLongSequence
- ///
- /// Values in 244..248 represent a too high codepoint, but do not cause an
- /// error.
- fn extra_utf8_bytes(self) -> Result<usize,InvalidUtf8FirstByte>;
-
- /// How many more bytes will you need to complete this codepoint?
- ///
- /// This function assumes that the byte is a valid UTF-8 start, and might
- /// return any value otherwise. (but the function is pure and safe to call
- /// with any value).
- fn extra_utf8_bytes_unchecked(self) -> usize;
-}
-
-impl U8UtfExt for u8 {
- #[inline]
- fn extra_utf8_bytes(self) -> Result<usize,InvalidUtf8FirstByte> {
- use error::InvalidUtf8FirstByte::{ContinuationByte,TooLongSeqence};
- // the bit twiddling is explained in extra_utf8_bytes_unchecked()
- if self < 128 {
- return Ok(0);
- }
- match ((self as u32)<<25).not().leading_zeros() {
- n @ 1...3 => Ok(n as usize),
- 0 => Err(ContinuationByte),
- _ => Err(TooLongSeqence),
- }
- }
- #[inline]
- fn extra_utf8_bytes_unchecked(self) -> usize {
- // For fun I've optimized this function (for x86 instruction count):
- // The most straightforward implementation, that lets the compiler do
- // the optimizing:
- //match self {
- // 0b0000_0000...0b0111_1111 => 0,
- // 0b1100_0010...0b1101_1111 => 1,
- // 0b1110_0000...0b1110_1111 => 2,
- // 0b1111_0000...0b1111_0100 => 3,
- // _ => whatever()
- //}
- // Using `unsafe{self::core::hint::unreachable_unchecked()}` for the
- // "don't care" case is a terrible idea: while having the function
- // non-deterministically return whatever happens to be in a register
- // MIGHT be acceptable, it permits the function to not `ret`urn at all,
- // but let execution fall through to whatever comes after it in the
- // binary! (in other words completely UB).
- // Currently unreachable_unchecked() might trap too,
- // which is certainly not what we want.
- // I also think `unsafe{mem::unitialized()}` is much more likely to
- // explicitly produce whatever happens to be in a register than tell
- // the compiler it can ignore this branch but needs to produce a value.
- //
- // From the bit patterns we see that for non-ASCII values the result is
- // (number of leading set bits) - 1
- // The standard library doesn't have a method for counting leading ones,
- // but it has leading_zeros(), which can be used after inverting.
- // This function can therefore be reduced to the one-liner
- //`self.not().leading_zeros().saturating_sub(1) as usize`, which would
- // be branchless for architectures with instructions for
- // leading_zeros() and saturating_sub().
-
- // Shortest version as long as ASCII-ness can be predicted: (especially
- // if the BSR instruction which leading_zeros() uses is microcoded or
- // doesn't exist)
- // u8.leading_zeros() would cast to a bigger type internally, so that's
- // free. compensating by shifting left by 24 before inverting lets the
- // compiler know that the value passed to leading_zeros() is not zero,
- // for which BSR's output is undefined and leading_zeros() normally has
- // special case with a branch.
- // Shifting one bit too many left acts as a saturating_sub(1).
- if self<128 {0} else {((self as u32)<<25).not().leading_zeros() as usize}
-
- // Branchless but longer version: (9 instructions)
- // It's tempting to try (self|0x80).not().leading_zeros().wrapping_sub(1),
- // but that produces high lengths for ASCII values 0b01xx_xxxx.
- // If we could somehow (branchlessy) clear that bit for ASCII values...
- // We can by masking with the value shifted right with sign extension!
- // (any nonzero number of bits in range works)
- //let extended = self as i8 as i32;
- //let ascii_cleared = (extended<<25) & (extended>>25);
- //ascii_cleared.not().leading_zeros() as usize
-
- // cmov version: (7 instructions)
- //(((self as u32)<<24).not().leading_zeros() as usize).saturating_sub(1)
- }
-}
-
-
-/// Methods for working with `u16`s as UTF-16 units.
-pub trait U16UtfExt {
- /// Will you need an extra unit to complete this codepoint?
- ///
- /// Returns `Err` for trailing surrogates, `Ok(true)` for leading surrogates,
- /// and `Ok(false)` for others.
- fn utf16_needs_extra_unit(self) -> Result<bool,InvalidUtf16FirstUnit>;
-
- /// Does this `u16` need another `u16` to complete a codepoint?
- /// Returns `(self & 0xfc00) == 0xd800`
- ///
- /// Is basically an unchecked variant of `utf16_needs_extra_unit()`.
- fn is_utf16_leading_surrogate(self) -> bool;
-}
-impl U16UtfExt for u16 {
- #[inline]
- fn utf16_needs_extra_unit(self) -> Result<bool,InvalidUtf16FirstUnit> {
- match self {
- // https://en.wikipedia.org/wiki/UTF-16#U.2B10000_to_U.2B10FFFF
- 0x00_00...0xd7_ff | 0xe0_00...0xff_ff => Ok(false),
- 0xd8_00...0xdb_ff => Ok(true),
- _ => Err(InvalidUtf16FirstUnit)
- }
- }
- #[inline]
- fn is_utf16_leading_surrogate(self) -> bool {
- (self & 0xfc00) == 0xd800// Clear the ten content bytes of a surrogate,
- // and see if it's a leading surrogate.
- }
-}
-
-
-
-
-/// Extension trait for `char` that adds methods for converting to and from UTF-8 or UTF-16.
-pub trait CharExt: Sized {
- /// Get the UTF-8 representation of this codepoint.
- ///
- /// `Utf8Char` is to `[u8;4]` what `char` is to `u32`:
- /// a restricted type that cannot be mutated internally.
- fn to_utf8(self) -> Utf8Char;
-
- /// Get the UTF-16 representation of this codepoint.
- ///
- /// `Utf16Char` is to `[u16;2]` what `char` is to `u32`:
- /// a restricted type that cannot be mutated internally.
- fn to_utf16(self) -> Utf16Char;
-
- /// Iterate over or [read](https://doc.rust-lang.org/std/io/trait.Read.html)
- /// the one to four bytes in the UTF-8 representation of this codepoint.
- ///
- /// An identical alternative to the unstable `char.encode_utf8()`.
- /// That method somehow still exist on stable, so I have to use a different name.
- fn iter_utf8_bytes(self) -> Utf8Iterator;
-
- /// Iterate over the one or two units in the UTF-16 representation of this codepoint.
- ///
- /// An identical alternative to the unstable `char.encode_utf16()`.
- /// That method somehow still exist on stable, so I have to use a different name.
- fn iter_utf16_units(self) -> Utf16Iterator;
-
-
- /// Convert this char to an UTF-8 array, and also return how many bytes of
- /// the array are used,
- ///
- /// The returned array is left-aligned with unused bytes set to zero.
- fn to_utf8_array(self) -> ([u8; 4], usize);
-
- /// Convert this `char` to UTF-16.
- ///
- /// The second element is non-zero when a surrogate pair is required.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::CharExt;
- ///
- /// assert_eq!('@'.to_utf16_array(), ['@' as u16, 0]);
- /// assert_eq!('睷'.to_utf16_array(), ['睷' as u16, 0]);
- /// assert_eq!('\u{abcde}'.to_utf16_array(), [0xda6f, 0xdcde]);
- /// ```
- fn to_utf16_array(self) -> [u16; 2];
-
- /// Convert this `char` to UTF-16.
- /// The second item is `Some` if a surrogate pair is required.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::CharExt;
- ///
- /// assert_eq!('@'.to_utf16_tuple(), ('@' as u16, None));
- /// assert_eq!('睷'.to_utf16_tuple(), ('睷' as u16, None));
- /// assert_eq!('\u{abcde}'.to_utf16_tuple(), (0xda6f, Some(0xdcde)));
- /// ```
- fn to_utf16_tuple(self) -> (u16, Option<u16>);
-
-
-
- /// Create a `char` from the start of an UTF-8 slice,
- /// and also return how many bytes were used.
- ///
- /// # Errors
- ///
- /// Returns an `Err` if the slice is empty, doesn't start with a valid
- /// UTF-8 sequence or is too short for the sequence.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::CharExt;
- /// use encode_unicode::error::InvalidUtf8Slice::*;
- /// use encode_unicode::error::InvalidUtf8::*;
- ///
- /// assert_eq!(char::from_utf8_slice_start(&[b'A', b'B', b'C']), Ok(('A',1)));
- /// assert_eq!(char::from_utf8_slice_start(&[0xdd, 0xbb]), Ok(('\u{77b}',2)));
- ///
- /// assert_eq!(char::from_utf8_slice_start(&[]), Err(TooShort(1)));
- /// assert_eq!(char::from_utf8_slice_start(&[0xf0, 0x99]), Err(TooShort(4)));
- /// assert_eq!(char::from_utf8_slice_start(&[0xee, b'F', 0x80]), Err(Utf8(NotAContinuationByte(1))));
- /// assert_eq!(char::from_utf8_slice_start(&[0xee, 0x99, 0x0f]), Err(Utf8(NotAContinuationByte(2))));
- /// ```
- fn from_utf8_slice_start(src: &[u8]) -> Result<(Self,usize),InvalidUtf8Slice>;
-
- /// Create a `char` from the start of an UTF-16 slice,
- /// and also return how many units were used.
- ///
- /// If you want to continue after an error, continue with the next `u16` unit.
- fn from_utf16_slice_start(src: &[u16]) -> Result<(Self,usize), InvalidUtf16Slice>;
-
-
- /// Convert an UTF-8 sequence as returned from `.to_utf8_array()` into a `char`
- ///
- /// The codepoint must start at the first byte, and leftover bytes are ignored.
- ///
- /// # Errors
- ///
- /// Returns an `Err` if the array doesn't start with a valid UTF-8 sequence.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::CharExt;
- /// use encode_unicode::error::InvalidUtf8Array::*;
- /// use encode_unicode::error::InvalidUtf8::*;
- /// use encode_unicode::error::InvalidCodepoint::*;
- ///
- /// assert_eq!(char::from_utf8_array([b'A', 0, 0, 0]), Ok('A'));
- /// assert_eq!(char::from_utf8_array([0xf4, 0x8b, 0xbb, 0xbb]), Ok('\u{10befb}'));
- /// assert_eq!(char::from_utf8_array([b'A', b'B', b'C', b'D']), Ok('A'));
- /// assert_eq!(char::from_utf8_array([0, 0, 0xcc, 0xbb]), Ok('\0'));
- ///
- /// assert_eq!(char::from_utf8_array([0xef, b'F', 0x80, 0x80]), Err(Utf8(NotAContinuationByte(1))));
- /// assert_eq!(char::from_utf8_array([0xc1, 0x80, 0, 0]), Err(Utf8(OverLong)));
- /// assert_eq!(char::from_utf8_array([0xf7, 0xaa, 0x99, 0x88]), Err(Codepoint(TooHigh)));
- /// ```
- fn from_utf8_array(utf8: [u8; 4]) -> Result<Self,InvalidUtf8Array>;
-
- /// Convert a UTF-16 pair as returned from `.to_utf16_array()` into a `char`.
- ///
- /// The second element is ignored when not required.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::CharExt;
- /// use encode_unicode::error::InvalidUtf16Array;
- ///
- /// assert_eq!(char::from_utf16_array(['x' as u16, 'y' as u16]), Ok('x'));
- /// assert_eq!(char::from_utf16_array(['睷' as u16, 0]), Ok('睷'));
- /// assert_eq!(char::from_utf16_array([0xda6f, 0xdcde]), Ok('\u{abcde}'));
- /// assert_eq!(char::from_utf16_array([0xf111, 0xdbad]), Ok('\u{f111}'));
- /// assert_eq!(char::from_utf16_array([0xdaaf, 0xdaaf]), Err(InvalidUtf16Array::SecondIsNotTrailingSurrogate));
- /// assert_eq!(char::from_utf16_array([0xdcac, 0x9000]), Err(InvalidUtf16Array::FirstIsTrailingSurrogate));
- /// ```
- fn from_utf16_array(utf16: [u16; 2]) -> Result<Self, InvalidUtf16Array>;
-
- /// Convert a UTF-16 pair as returned from `.to_utf16_tuple()` into a `char`.
- fn from_utf16_tuple(utf16: (u16, Option<u16>)) -> Result<Self, InvalidUtf16Tuple>;
-
-
- /// Convert an UTF-8 sequence into a char.
- ///
- /// The length of the slice is taken as length of the sequence;
- /// it should be 1,2,3 or 4.
- ///
- /// # Safety
- ///
- /// The slice must contain exactly one, valid, UTF-8 sequence.
- ///
- /// Passing a slice that produces an invalid codepoint is always undefined
- /// behavior; Later checks that the codepoint is valid can be removed
- /// by the compiler.
- ///
- /// # Panics
- ///
- /// If the slice is empty
- unsafe fn from_utf8_exact_slice_unchecked(src: &[u8]) -> Self;
-
- /// Convert a UTF-16 array as returned from `.to_utf16_array()` into a
- /// `char`.
- ///
- /// This function is safe because it avoids creating invalid codepoints,
- /// but the returned value might not be what one expectedd.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::CharExt;
- ///
- /// // starts with a trailing surrogate - converted as if it was a valid
- /// // surrogate pair anyway.
- /// assert_eq!(char::from_utf16_array_unchecked([0xdbad, 0xf19e]), '\u{fb59e}');
- /// // missing trailing surrogate - ditto
- /// assert_eq!(char::from_utf16_array_unchecked([0xd802, 0]), '\u{10800}');
- /// ```
- fn from_utf16_array_unchecked(utf16: [u16;2]) -> Self;
-
- /// Convert a UTF-16 tuple as returned from `.to_utf16_tuple()` into a `char`.
- unsafe fn from_utf16_tuple_unchecked(utf16: (u16, Option<u16>)) -> Self;
-
-
- /// Produces more detailed errors than `char::from_u32()`
- ///
- /// # Errors
- ///
- /// This function will return an error if
- ///
- /// * the value is greater than 0x10ffff
- /// * the value is between 0xd800 and 0xdfff (inclusive)
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::CharExt;
- /// use encode_unicode::error::InvalidCodepoint;
- ///
- /// assert_eq!(char::from_u32_detailed(0x41), Ok('A'));
- /// assert_eq!(char::from_u32_detailed(0x40_00_00), Err(InvalidCodepoint::TooHigh));
- /// assert_eq!(char::from_u32_detailed(0xd951), Err(InvalidCodepoint::Utf16Reserved));
- /// assert_eq!(char::from_u32_detailed(0xdddd), Err(InvalidCodepoint::Utf16Reserved));
- /// assert_eq!(char::from_u32_detailed(0xdd), Ok('Ý'));
- /// assert_eq!(char::from_u32_detailed(0x1f331), Ok('🌱'));
- /// ```
- fn from_u32_detailed(c: u32) -> Result<Self,InvalidCodepoint>;
-}
-
-
-
-impl CharExt for char {
- /////////
- //UTF-8//
- /////////
-
- fn to_utf8(self) -> Utf8Char {
- self.into()
- }
- fn iter_utf8_bytes(self) -> Utf8Iterator {
- self.to_utf8().into_iter()
- }
-
- fn to_utf8_array(self) -> ([u8; 4], usize) {
- let len = self.len_utf8();
- let mut c = self as u32;
- if len == 1 {// ASCII, the common case
- ([c as u8, 0, 0, 0], 1)
- } else {
- let mut parts = 0;// convert to 6-bit bytes
- parts |= c & 0x3f; c>>=6;
- parts<<=8; parts |= c & 0x3f; c>>=6;
- parts<<=8; parts |= c & 0x3f; c>>=6;
- parts<<=8; parts |= c & 0x3f;
- parts |= 0x80_80_80_80;// set the most significant bit
- parts >>= 8*(4-len);// right-align bytes
- // Now, unused bytes are zero, (which matters for Utf8Char.eq())
- // and the rest are 0b10xx_xxxx
-
- // set header on first byte
- parts |= (0xff_00u32 >> len) & 0xff;// store length
- parts &= Not::not(1u32 << 7-len);// clear the next bit after it
-
- let bytes: [u8; 4] = unsafe{ mem::transmute(u32::from_le(parts)) };
- (bytes, len)
- }
- }
-
-
- fn from_utf8_slice_start(src: &[u8]) -> Result<(Self,usize),InvalidUtf8Slice> {
- use errors::InvalidUtf8::*;
- use errors::InvalidUtf8Slice::*;
- let first = match src.first() {
- Some(first) => *first,
- None => return Err(TooShort(1)),
- };
- let bytes = match first.extra_utf8_bytes() {
- Err(e) => return Err(Utf8(FirstByte(e))),
- Ok(0) => return Ok((first as char, 1)),
- Ok(extra) if extra >= src.len()
- => return Err(TooShort(extra+1)),
- Ok(extra) => &src[..extra+1],
- };
- if let Some(i) = bytes.iter().skip(1).position(|&b| (b >> 6) != 0b10 ) {
- Err(Utf8(NotAContinuationByte(i+1)))
- } else if overlong(bytes[0], bytes[1]) {
- Err(Utf8(OverLong))
- } else {
- match char::from_u32_detailed(merge_nonascii_unchecked_utf8(bytes)) {
- Ok(c) => Ok((c, bytes.len())),
- Err(e) => Err(Codepoint(e)),
- }
- }
- }
-
- fn from_utf8_array(utf8: [u8; 4]) -> Result<Self,InvalidUtf8Array> {
- use errors::InvalidUtf8::*;
- use errors::InvalidUtf8Array::*;
- let src = match utf8[0].extra_utf8_bytes() {
- Err(error) => return Err(Utf8(FirstByte(error))),
- Ok(0) => return Ok(utf8[0] as char),
- Ok(extra) => &utf8[..extra+1],
- };
- if let Some(i) = src[1..].iter().position(|&b| (b >> 6) != 0b10 ) {
- Err(Utf8(NotAContinuationByte(i+1)))
- } else if overlong(utf8[0], utf8[1]) {
- Err(Utf8(OverLong))
- } else {
- char::from_u32_detailed(merge_nonascii_unchecked_utf8(src))
- .map_err(|e| Codepoint(e) )
- }
- }
-
- unsafe fn from_utf8_exact_slice_unchecked(src: &[u8]) -> Self {
- if src.len() == 1 {
- src[0] as char
- } else {
- char::from_u32_unchecked(merge_nonascii_unchecked_utf8(src))
- }
- }
-
-
-
- //////////
- //UTF-16//
- //////////
-
- fn to_utf16(self) -> Utf16Char {
- Utf16Char::from(self)
- }
- fn iter_utf16_units(self) -> Utf16Iterator {
- self.to_utf16().into_iter()
- }
-
- fn to_utf16_array(self) -> [u16;2] {
- let (first, second) = self.to_utf16_tuple();
- [first, second.unwrap_or(0)]
- }
- fn to_utf16_tuple(self) -> (u16, Option<u16>) {
- if self <= '\u{ffff}' {// single
- (self as u16, None)
- } else {// double
- let c = self as u32 - 0x_01_00_00;
- let high = 0x_d8_00 + (c >> 10);
- let low = 0x_dc_00 + (c & 0x_03_ff);
- (high as u16, Some(low as u16))
- }
- }
-
-
- fn from_utf16_slice_start(src: &[u16]) -> Result<(Self,usize), InvalidUtf16Slice> {
- use errors::InvalidUtf16Slice::*;
- unsafe {match (src.get(0), src.get(1)) {
- (Some(&u @ 0x00_00...0xd7_ff), _) |
- (Some(&u @ 0xe0_00...0xff_ff), _)
- => Ok((char::from_u32_unchecked(u as u32), 1)),
- (Some(&0xdc_00...0xdf_ff), _) => Err(FirstLowSurrogate),
- (None, _) => Err(EmptySlice),
- (Some(&f @ 0xd8_00...0xdb_ff), Some(&s @ 0xdc_00...0xdf_ff))
- => Ok((char::from_utf16_tuple_unchecked((f, Some(s))), 2)),
- (Some(&0xd8_00...0xdb_ff), Some(_)) => Err(SecondNotLowSurrogate),
- (Some(&0xd8_00...0xdb_ff), None) => Err(MissingSecond),
- (Some(_), _) => unreachable!()
- }}
- }
-
- fn from_utf16_array(utf16: [u16;2]) -> Result<Self, InvalidUtf16Array> {
- use errors::InvalidUtf16Array::*;
- if let Some(c) = char::from_u32(utf16[0] as u32) {
- Ok(c) // single
- } else if utf16[0] < 0xdc_00 && utf16[1] & 0xfc_00 == 0xdc_00 {
- // correct surrogate pair
- Ok(combine_surrogates(utf16[0], utf16[1]))
- } else if utf16[0] < 0xdc_00 {
- Err(SecondIsNotTrailingSurrogate)
- } else {
- Err(FirstIsTrailingSurrogate)
- }
- }
- fn from_utf16_tuple(utf16: (u16, Option<u16>)) -> Result<Self, InvalidUtf16Tuple> {
- use errors::InvalidUtf16Tuple::*;
- unsafe{ match utf16 {
- (0x00_00...0xd7_ff, None) | // single
- (0xe0_00...0xff_ff, None) | // single
- (0xd8_00...0xdb_ff, Some(0xdc_00...0xdf_ff)) // correct surrogate
- => Ok(char::from_utf16_tuple_unchecked(utf16)),
- (0xd8_00...0xdb_ff, Some(_)) => Err(InvalidSecond),
- (0xd8_00...0xdb_ff, None ) => Err(MissingSecond),
- (0xdc_00...0xdf_ff, _ ) => Err(FirstIsTrailingSurrogate),
- ( _ , Some(_)) => Err(SuperfluousSecond),
- ( _ , None ) => unreachable!()
- }}
- }
-
- fn from_utf16_array_unchecked(utf16: [u16;2]) -> Self {
- // treat any array with a surrogate value in [0] as a surrogate because
- // combine_surrogates() is safe.
- // `(utf16[0] & 0xf800) == 0xd80` might not be quite as fast as
- // `utf16[1] != 0`, but avoiding the potential for UB is worth it
- // since the conversion isn't zero-cost in either case.
- char::from_u32(utf16[0] as u32)
- .unwrap_or_else(|| combine_surrogates(utf16[0], utf16[1]) )
- }
- unsafe fn from_utf16_tuple_unchecked(utf16: (u16, Option<u16>)) -> Self {
- match utf16.1 {
- Some(second) => combine_surrogates(utf16.0, second),
- None => char::from_u32_unchecked(utf16.0 as u32)
- }
- }
-
-
- fn from_u32_detailed(c: u32) -> Result<Self,InvalidCodepoint> {
- match char::from_u32(c) {
- Some(c) => Ok(c),
- None if c > 0x10_ff_ff => Err(InvalidCodepoint::TooHigh),
- None => Err(InvalidCodepoint::Utf16Reserved),
- }
- }
-}
-
-// Adapted from https://www.cl.cam.ac.uk/~mgk25/ucs/utf8_check.c
-fn overlong(first: u8, second: u8) -> bool {
- if first < 0x80 {
- false
- } else if (first & 0xe0) == 0xc0 {
- (first & 0xfe) == 0xc0
- } else if (first & 0xf0) == 0xe0 {
- first == 0xe0 && (second & 0xe0) == 0x80
- } else {
- first == 0xf0 && (second & 0xf0) == 0x80
- }
-}
-
-/// Decodes the codepoint represented by a multi-byte UTF-8 sequence.
-///
-/// Does not check that the codepoint is valid,
-/// and returns `u32` because casting invalid codepoints to `char` is insta UB.
-fn merge_nonascii_unchecked_utf8(src: &[u8]) -> u32 {
- let mut c = src[0] as u32 & (0x7f >> src.len());
- for b in &src[1..] {
- c = (c << 6) | (b & 0b0011_1111) as u32;
- }
- c
-}
-
-/// Create a `char` from a leading and a trailing surrogate.
-///
-/// This function is safe because it ignores the six most significant bits of
-/// each arguments and always produces a codepoint in 0x01_00_00..=0x10_ff_ff.
-fn combine_surrogates(first: u16, second: u16) -> char {
- unsafe {
- let high = (first & 0x_03_ff) as u32;
- let low = (second & 0x_03_ff) as u32;
- let c = ((high << 10) | low) + 0x_01_00_00; // no, the constant can't be or'd in
- char::from_u32_unchecked(c)
- }
-}
-
-
-
-/// Adds `.utf8chars()` and `.utf16chars()` iterator constructors to `&str`.
-pub trait StrExt: AsRef<str> {
- /// Equivalent to `.chars()` but produces `Utf8Char`s.
- fn utf8chars(&self) -> Utf8Chars;
- /// Equivalent to `.chars()` but produces `Utf16Char`s.
- fn utf16chars(&self) -> Utf16Chars;
- /// Equivalent to `.char_indices()` but produces `Utf8Char`s.
- fn utf8char_indices(&self) -> Utf8CharIndices;
- /// Equivalent to `.char_indices()` but produces `Utf16Char`s.
- fn utf16char_indices(&self) -> Utf16CharIndices;
-}
-
-impl StrExt for str {
- fn utf8chars(&self) -> Utf8Chars {
- Utf8Chars::from(self)
- }
- fn utf16chars(&self) -> Utf16Chars {
- Utf16Chars::from(self)
- }
- fn utf8char_indices(&self) -> Utf8CharIndices {
- Utf8CharIndices::from(self)
- }
- fn utf16char_indices(&self) -> Utf16CharIndices {
- Utf16CharIndices::from(self)
- }
-}
-
-#[cfg(feature="ascii")]
-impl StrExt for AsciiStr {
- fn utf8chars(&self) -> Utf8Chars {
- Utf8Chars::from(self.as_str())
- }
- fn utf16chars(&self) -> Utf16Chars {
- Utf16Chars::from(self.as_str())
- }
- fn utf8char_indices(&self) -> Utf8CharIndices {
- Utf8CharIndices::from(self.as_str())
- }
- fn utf16char_indices(&self) -> Utf16CharIndices {
- Utf16CharIndices::from(self.as_str())
- }
-}
-
-
-
-/// Iterator methods that convert between `u8`s and `Utf8Char` or `u16`s and `Utf16Char`
-///
-/// All the iterator adapters also accept iterators that produce references of
-/// the type they convert from.
-pub trait IterExt: Iterator+Sized {
- /// Converts an iterator of `Utf8Char`s or `&Utf8Char`s to an iterator of
- /// `u8`s.
- ///
- /// Has the same effect as `.flat_map()` or `.flatten()`, but the returned
- /// iterator is ~40% faster.
- ///
- /// The iterator also implements `Read`
- /// (when the `std` feature isn't disabled).
- /// Reading will never produce an error, and calls to `.read()` and `.next()`
- /// can be mixed.
- ///
- /// The exact number of bytes cannot be known in advance, but `size_hint()`
- /// gives the possible range.
- /// (min: all remaining characters are ASCII, max: all require four bytes)
- ///
- /// # Examples
- ///
- /// From iterator of values:
- ///
- /// ```
- /// use encode_unicode::{IterExt, StrExt};
- ///
- /// let iterator = "foo".utf8chars();
- /// let mut bytes = [0; 4];
- /// for (u,dst) in iterator.to_bytes().zip(&mut bytes) {*dst=u;}
- /// assert_eq!(&bytes, b"foo\0");
- /// ```
- ///
- /// From iterator of references:
- ///
- #[cfg_attr(feature="std", doc=" ```")]
- #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
- /// use encode_unicode::{IterExt, StrExt, Utf8Char};
- ///
- /// let chars: Vec<Utf8Char> = "💣 bomb 💣".utf8chars().collect();
- /// let bytes: Vec<u8> = chars.iter().to_bytes().collect();
- /// let flat_map: Vec<u8> = chars.iter().flat_map(|u8c| *u8c ).collect();
- /// assert_eq!(bytes, flat_map);
- /// ```
- ///
- /// `Read`ing from it:
- ///
- #[cfg_attr(feature="std", doc=" ```")]
- #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
- /// use encode_unicode::{IterExt, StrExt};
- /// use std::io::Read;
- ///
- /// let s = "Ååh‽";
- /// assert_eq!(s.len(), 8);
- /// let mut buf = [b'E'; 9];
- /// let mut reader = s.utf8chars().to_bytes();
- /// assert_eq!(reader.read(&mut buf[..]).unwrap(), 8);
- /// assert_eq!(reader.read(&mut buf[..]).unwrap(), 0);
- /// assert_eq!(&buf[..8], s.as_bytes());
- /// assert_eq!(buf[8], b'E');
- /// ```
- fn to_bytes(self) -> Utf8CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf8Char>;
-
- /// Converts an iterator of `Utf16Char` (or `&Utf16Char`) to an iterator of
- /// `u16`s.
- ///
- /// Has the same effect as `.flat_map()` or `.flatten()`, but the returned
- /// iterator is about twice as fast.
- ///
- /// The exact number of units cannot be known in advance, but `size_hint()`
- /// gives the possible range.
- ///
- /// # Examples
- ///
- /// From iterator of values:
- ///
- /// ```
- /// use encode_unicode::{IterExt, StrExt};
- ///
- /// let iterator = "foo".utf16chars();
- /// let mut units = [0; 4];
- /// for (u,dst) in iterator.to_units().zip(&mut units) {*dst=u;}
- ///
- /// assert_eq!(units, ['f' as u16, 'o' as u16, 'o' as u16, 0]);
- /// ```
- ///
- /// From iterator of references:
- ///
- #[cfg_attr(feature="std", doc=" ```")]
- #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
- /// use encode_unicode::{IterExt, StrExt, Utf16Char};
- ///
- /// // (💣 takes two units)
- /// let chars: Vec<Utf16Char> = "💣 bomb 💣".utf16chars().collect();
- /// let units: Vec<u16> = chars.iter().to_units().collect();
- /// let flat_map: Vec<u16> = chars.iter().flat_map(|u16c| *u16c ).collect();
- ///
- /// assert_eq!(units, flat_map);
- /// ```
- fn to_units(self) -> Utf16CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf16Char>;
-
- /// Decodes bytes as UTF-8 and groups them into `Utf8Char`s
- ///
- /// When errors (invalid values or sequences) are encountered,
- /// it continues with the byte right after the start of the error sequence.
- /// This is neither the most intelligent choiche (sometimes it is guaranteed to
- /// produce another error), nor the easiest to implement, but I believe it to
- /// be the most predictable.
- /// It also means that ASCII characters are never hidden by errors.
- ///
- /// # Examples
- ///
- /// Replace all errors with u+FFFD REPLACEMENT_CHARACTER:
- /// ```
- /// use encode_unicode::{Utf8Char, IterExt};
- ///
- /// let mut buf = [b'\0'; 255];
- /// let len = b"foo\xCFbar".iter()
- /// .to_utf8chars()
- /// .flat_map(|r| r.unwrap_or(Utf8Char::from('\u{FFFD}')).into_iter() )
- /// .zip(&mut buf[..])
- /// .map(|(byte, dst)| *dst = byte )
- /// .count();
- ///
- /// assert_eq!(&buf[..len], "foo\u{FFFD}bar".as_bytes());
- /// ```
- ///
- /// Collect everything up until the first error into a string:
- #[cfg_attr(feature="std", doc=" ```")]
- #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
- /// use encode_unicode::iterator::Utf8CharMerger;
- /// let mut good = String::new();
- /// for r in Utf8CharMerger::from(b"foo\xcc\xbbbar\xcc\xddbaz") {
- /// if let Ok(uc) = r {
- /// good.push_str(uc.as_str());
- /// } else {
- /// break;
- /// }
- /// }
- /// assert_eq!(good, "foo̻bar");
- /// ```
- ///
- /// Abort decoding on error:
- #[cfg_attr(feature="std", doc=" ```")]
- #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
- /// use encode_unicode::{IterExt, Utf8Char};
- /// use encode_unicode::error::{InvalidUtf8Slice, InvalidUtf8};
- ///
- /// let result = b"ab\0\xe0\xbc\xa9 \xf3\x80\x77".iter()
- /// .to_utf8chars()
- /// .collect::<Result<String,InvalidUtf8Slice>>();
- ///
- /// assert_eq!(result, Err(InvalidUtf8Slice::Utf8(InvalidUtf8::NotAContinuationByte(2))));
- /// ```
- fn to_utf8chars(self) -> Utf8CharMerger<Self::Item,Self> where Self::Item: Borrow<u8>;
-
- /// Decodes bytes as UTF-16 and groups them into `Utf16Char`s
- ///
- /// When errors (unmatched leading surrogates or unexpected trailing surrogates)
- /// are encountered, an error is produced for every unit.
- ///
- /// # Examples
- ///
- /// Replace errors with '�':
- #[cfg_attr(feature="std", doc=" ```")]
- #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
- /// use encode_unicode::{IterExt, Utf16Char};
- ///
- /// let slice = &['a' as u16, 0xdf00, 0xd83c, 0xdca0][..];
- /// let string = slice.iter()
- /// .to_utf16chars()
- /// .map(|r| r.unwrap_or(Utf16Char::from('\u{fffd}')) ) // REPLACEMENT_CHARACTER
- /// .collect::<String>();
- ///
- /// assert_eq!(string, "a�🂠");
- /// ```
- ///
- /// ```
- /// use encode_unicode::{IterExt, Utf16Char};
- /// use encode_unicode::error::Utf16PairError::*;
- ///
- /// let slice = [0xdcba, 0xdeff, 0xd8be, 0xdeee, 'Y' as u16, 0xdab1, 0xdab1];
- /// let mut iter = slice.iter().to_utf16chars();
- /// assert_eq!(iter.size_hint(), (3, Some(7)));
- /// assert_eq!(iter.next(), Some(Err(UnexpectedTrailingSurrogate)));
- /// assert_eq!(iter.next(), Some(Err(UnexpectedTrailingSurrogate)));
- /// assert_eq!(iter.next(), Some(Ok(Utf16Char::from('\u{3faee}'))));
- /// assert_eq!(iter.next(), Some(Ok(Utf16Char::from('Y'))));
- /// assert_eq!(iter.next(), Some(Err(UnmatchedLeadingSurrogate)));
- /// assert_eq!(iter.next(), Some(Err(Incomplete)));
- /// assert_eq!(iter.into_remaining_units().next(), None);
- /// ```
- ///
- /// Search for a codepoint and return the codepoint index of the first match:
- /// ```
- /// use encode_unicode::{IterExt, Utf16Char};
- ///
- /// let position = [0xd875, 0xdd4f, '≈' as u16, '2' as u16].iter()
- /// .to_utf16chars()
- /// .position(|r| r == Ok(Utf16Char::from('≈')) );
- ///
- /// assert_eq!(position, Some(1));
- /// ```
- fn to_utf16chars(self) -> Utf16CharMerger<Self::Item,Self> where Self::Item: Borrow<u16>;
-}
-
-impl<I:Iterator> IterExt for I {
- fn to_bytes(self) -> Utf8CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf8Char> {
- iter_bytes(self)
- }
- fn to_units(self) -> Utf16CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf16Char> {
- iter_units(self)
- }
- fn to_utf8chars(self) -> Utf8CharMerger<Self::Item,Self> where Self::Item: Borrow<u8> {
- Utf8CharMerger::from(self)
- }
- fn to_utf16chars(self) -> Utf16CharMerger<Self::Item,Self> where Self::Item: Borrow<u16> {
- Utf16CharMerger::from(self)
- }
-}
-
-
-/// Methods for iterating over `u8` and `u16` slices as UTF-8 or UTF-16 characters.
-///
-/// The iterators are slightly faster than the similar methods in [`IterExt`](trait.IterExt.html)
-/// because they con "push back" items for free after errors and don't need a
-/// separate buffer that must be checked on every call to `.next()`.
-pub trait SliceExt: Index<RangeFull> {
- /// Decode `u8` slices as UTF-8 and iterate over the codepoints as `Utf8Char`s,
- ///
- /// # Examples
- ///
- /// Get the index and error type of the first error:
- #[cfg_attr(feature="std", doc=" ```")]
- #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
- /// use encode_unicode::{SliceExt, Utf8Char};
- /// use encode_unicode::error::InvalidUtf8Slice;
- ///
- /// let slice = b"ab\0\xe0\xbc\xa9 \xf3\x80\x77";
- /// let result = slice.utf8char_indices()
- /// .map(|(offset,r,length)| r.map_err(|e| (offset,e,length) ) )
- /// .collect::<Result<String,(usize,InvalidUtf8Slice,usize)>>();
- ///
- /// assert_eq!(result, Err((7, InvalidUtf8Slice::TooShort(4), 1)));
- /// ```
- ///
- /// ```
- /// use encode_unicode::{SliceExt, Utf8Char};
- /// use std::error::Error;
- ///
- /// let slice = b"\xf0\xbf\xbf\xbfXY\xdd\xbb\xe1\x80\x99quux123";
- /// let mut fixed_size = [Utf8Char::default(); 8];
- /// for (cp_i, (byte_index, r, _)) in slice.utf8char_indices().enumerate().take(8) {
- /// match r {
- /// Ok(u8c) => fixed_size[cp_i] = u8c,
- /// Err(e) => panic!("Invalid codepoint at index {} ({})", cp_i, e.description()),
- /// }
- /// }
- /// let chars = ['\u{3ffff}', 'X', 'Y', '\u{77b}', '\u{1019}', 'q', 'u', 'u'];
- /// assert_eq!(fixed_size, chars);
- /// ```
- ///
- #[cfg_attr(feature="std", doc=" ```")]
- #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
- /// use encode_unicode::{SliceExt, Utf8Char};
- /// use encode_unicode::error::InvalidUtf8Slice::*;
- /// use encode_unicode::error::{InvalidUtf8, InvalidUtf8FirstByte, InvalidCodepoint};
- ///
- /// let bytes = b"\xfa-\xf4\x8f\xee\xa1\x8f-\xed\xa9\x87\xf0\xcc\xbb";
- /// let mut errors = Vec::new();
- /// let mut lengths = Vec::new();
- /// let mut string = String::new();
- /// for (offset,result,length) in bytes.utf8char_indices() {
- /// lengths.push((offset,length));
- /// let c = result.unwrap_or_else(|error| {
- /// errors.push((offset,error));
- /// Utf8Char::from('\u{fffd}') // replacement character
- /// });
- /// string.push_str(c.as_str());
- /// }
- ///
- /// assert_eq!(string, "�-��\u{e84f}-����\u{33b}");
- /// assert_eq!(lengths, [(0,1), (1,1), (2,1), (3,1), (4,3), (7,1),
- /// (8,1), (9,1), (10,1), (11,1), (12,2)]);
- /// assert_eq!(errors, [
- /// ( 0, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::TooLongSeqence))),
- /// ( 2, Utf8(InvalidUtf8::NotAContinuationByte(2))),
- /// ( 3, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::ContinuationByte))),
- /// ( 8, Codepoint(InvalidCodepoint::Utf16Reserved)),
- /// ( 9, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::ContinuationByte))),
- /// (10, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::ContinuationByte))),
- /// (11, TooShort(4)), // (but it was not the last element returned!)
- /// ]);
- /// ```
- fn utf8char_indices(&self) -> Utf8CharDecoder where Self::Output: Borrow<[u8]>;
-
-
- /// Decode `u16` slices as UTF-16 and iterate over the codepoints as `Utf16Char`s,
- ///
- /// The iterator produces `(usize,Result<Utf16Char,Utf16Error>,usize)`,
- /// and the slice is validated as you go.
- ///
- /// The first `usize` contains the offset from the start of the slice and
- /// the last `usize` contains the length of the codepoint or error.
- /// The length is either 1 or 2, and always 1 for errors.
- ///
- /// # Examples
- ///
- #[cfg_attr(feature="std", doc=" ```")]
- #[cfg_attr(not(feature="std"), doc=" ```no_compile")]
- /// use encode_unicode::{SliceExt, Utf8Char};
- ///
- /// let slice = &['a' as u16, 0xdf00, 0xd83c, 0xdca0][..];
- /// let mut errors = Vec::new();
- /// let string = slice.utf16char_indices().map(|(offset,r,_)| match r {
- /// Ok(u16c) => Utf8Char::from(u16c),
- /// Err(_) => {
- /// errors.push(offset);
- /// Utf8Char::from('\u{fffd}') // REPLACEMENT_CHARACTER
- /// }
- /// }).collect::<String>();
- ///
- /// assert_eq!(string, "a�🂠");
- /// assert_eq!(errors, [1]);
- /// ```
- ///
- /// Search for a codepoint and return its unit and codepoint index.
- /// ```
- /// use encode_unicode::{SliceExt, Utf16Char};
- ///
- /// let slice = [0xd875,/*'𝕏'*/ 0xdd4f, '≈' as u16, '2' as u16];
- /// let position = slice.utf16char_indices()
- /// .enumerate()
- /// .find(|&(_,(_,r,_))| r == Ok(Utf16Char::from('≈')) )
- /// .map(|(codepoint, (offset, _, _))| (codepoint, offset) );
- ///
- /// assert_eq!(position, Some((1,2)));
- /// ```
- ///
- /// Error types:
- /// ```
- /// use encode_unicode::{SliceExt, Utf16Char};
- /// use encode_unicode::error::Utf16PairError::*;
- ///
- /// let slice = [0xdcba, 0xdeff, 0xd8be, 0xdeee, 'λ' as u16, 0xdab1, 0xdab1];
- /// let mut iter = slice.utf16char_indices();
- /// assert_eq!(iter.next(), Some((0, Err(UnexpectedTrailingSurrogate), 1)));
- /// assert_eq!(iter.next(), Some((1, Err(UnexpectedTrailingSurrogate), 1)));
- /// assert_eq!(iter.next(), Some((2, Ok(Utf16Char::from('\u{3faee}')), 2)));
- /// assert_eq!(iter.next(), Some((4, Ok(Utf16Char::from('λ')), 1)));
- /// assert_eq!(iter.next(), Some((5, Err(UnmatchedLeadingSurrogate), 1)));
- /// assert_eq!(iter.next(), Some((6, Err(Incomplete), 1)));
- /// assert_eq!(iter.next(), None);
- /// assert_eq!(iter.as_slice(), [])
- /// ```
- fn utf16char_indices(&self) -> Utf16CharDecoder where Self::Output: Borrow<[u16]>;
-}
-
-impl<S: ?Sized+Index<RangeFull>> SliceExt for S {
- fn utf8char_indices(&self) -> Utf8CharDecoder where Self::Output: Borrow<[u8]> {
- Utf8CharDecoder::from(self[..].borrow())
- }
- fn utf16char_indices(&self) -> Utf16CharDecoder where Self::Output: Borrow<[u16]> {
- Utf16CharDecoder::from(self[..].borrow())
- }
-}
diff --git a/vendor/encode_unicode/src/utf16_char.rs b/vendor/encode_unicode/src/utf16_char.rs
deleted file mode 100644
index d8a6cec..0000000
--- a/vendor/encode_unicode/src/utf16_char.rs
+++ /dev/null
@@ -1,687 +0,0 @@
-/* Copyright 2016 The encode_unicode Developers
- *
- * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
- * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
- * http://opensource.org/licenses/MIT>, at your option. This file may not be
- * copied, modified, or distributed except according to those terms.
- */
-
-use utf16_iterators::Utf16Iterator;
-use traits::{CharExt, U16UtfExt};
-use utf8_char::Utf8Char;
-use errors::{InvalidUtf16Slice, InvalidUtf16Array, InvalidUtf16Tuple};
-use errors::{NonBMPError, EmptyStrError, FromStrError};
-extern crate core;
-use self::core::{hash,fmt};
-use self::core::cmp::Ordering;
-use self::core::borrow::Borrow;
-use self::core::ops::Deref;
-use self::core::str::FromStr;
-#[cfg(feature="std")]
-use self::core::iter::FromIterator;
-#[cfg(feature="std")]
-#[allow(deprecated)]
-use std::ascii::AsciiExt;
-#[cfg(feature="ascii")]
-use self::core::char;
-#[cfg(feature="ascii")]
-extern crate ascii;
-#[cfg(feature="ascii")]
-use self::ascii::{AsciiChar,ToAsciiChar,ToAsciiCharError};
-
-
-// I don't think there is any good default value for char, but char does.
-#[derive(Default)]
-// char doesn't do anything more advanced than u32 for Eq/Ord, so we shouldn't either.
-// When it's a single unit, the second is zero, so Eq works.
-// #[derive(Ord)] however, breaks on surrogate pairs.
-#[derive(PartialEq,Eq)]
-#[derive(Clone,Copy)]
-
-
-/// An unicode codepoint stored as UTF-16.
-///
-/// It can be borrowed as an `u16` slice, and has the same size as `char`.
-pub struct Utf16Char {
- units: [u16; 2],
-}
-
-
- /////////////////////
- //conversion traits//
-/////////////////////
-impl FromStr for Utf16Char {
- type Err = FromStrError;
- /// Create an `Utf16Char` from a string slice.
- /// The string must contain exactly one codepoint.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::error::FromStrError::*;
- /// use encode_unicode::Utf16Char;
- /// use std::str::FromStr;
- ///
- /// assert_eq!(Utf16Char::from_str("a"), Ok(Utf16Char::from('a')));
- /// assert_eq!(Utf16Char::from_str("🂠"), Ok(Utf16Char::from('🂠')));
- /// assert_eq!(Utf16Char::from_str(""), Err(Empty));
- /// assert_eq!(Utf16Char::from_str("ab"), Err(MultipleCodepoints));
- /// assert_eq!(Utf16Char::from_str("é"), Err(MultipleCodepoints));// 'e'+u301 combining mark
- /// ```
- fn from_str(s: &str) -> Result<Self, FromStrError> {
- match Utf16Char::from_str_start(s) {
- Ok((u16c,bytes)) if bytes == s.len() => Ok(u16c),
- Ok((_,_)) => Err(FromStrError::MultipleCodepoints),
- Err(EmptyStrError) => Err(FromStrError::Empty),
- }
- }
-}
-impl From<char> for Utf16Char {
- fn from(c: char) -> Self {
- let (first, second) = c.to_utf16_tuple();
- Utf16Char{ units: [first, second.unwrap_or(0)] }
- }
-}
-impl From<Utf8Char> for Utf16Char {
- fn from(utf8: Utf8Char) -> Utf16Char {
- let (b, utf8_len) = utf8.to_array();
- match utf8_len {
- 1 => Utf16Char{ units: [b[0] as u16, 0] },
- 4 => {// need surrogate
- let mut first = 0xd800 - (0x01_00_00u32 >> 10) as u16;
- first += (b[0] as u16 & 0x07) << 8;
- first += (b[1] as u16 & 0x3f) << 2;
- first += (b[2] as u16 & 0x30) >> 4;
- let mut second = 0xdc00;
- second |= (b[2] as u16 & 0x0f) << 6;
- second |= b[3] as u16 & 0x3f;
- Utf16Char{ units: [first, second] }
- },
- _ => { // 2 or 3
- let mut unit = ((b[0] as u16 & 0x1f) << 6) | (b[1] as u16 & 0x3f);
- if utf8_len == 3 {
- unit = (unit << 6) | (b[2] as u16 & 0x3f);
- }
- Utf16Char{ units: [unit, 0] }
- },
- }
- }
-}
-impl From<Utf16Char> for char {
- fn from(uc: Utf16Char) -> char {
- char::from_utf16_array_unchecked(uc.to_array())
- }
-}
-impl IntoIterator for Utf16Char {
- type Item=u16;
- type IntoIter=Utf16Iterator;
- /// Iterate over the units.
- fn into_iter(self) -> Utf16Iterator {
- Utf16Iterator::from(self)
- }
-}
-
-#[cfg(feature="std")]
-impl Extend<Utf16Char> for Vec<u16> {
- fn extend<I:IntoIterator<Item=Utf16Char>>(&mut self, iter: I) {
- let iter = iter.into_iter();
- self.reserve(iter.size_hint().0);
- for u16c in iter {
- self.push(u16c.units[0]);
- if u16c.units[1] != 0 {
- self.push(u16c.units[1]);
- }
- }
- }
-}
-#[cfg(feature="std")]
-impl<'a> Extend<&'a Utf16Char> for Vec<u16> {
- fn extend<I:IntoIterator<Item=&'a Utf16Char>>(&mut self, iter: I) {
- self.extend(iter.into_iter().cloned())
- }
-}
-#[cfg(feature="std")]
-impl FromIterator<Utf16Char> for Vec<u16> {
- fn from_iter<I:IntoIterator<Item=Utf16Char>>(iter: I) -> Self {
- let mut vec = Vec::new();
- vec.extend(iter);
- return vec;
- }
-}
-#[cfg(feature="std")]
-impl<'a> FromIterator<&'a Utf16Char> for Vec<u16> {
- fn from_iter<I:IntoIterator<Item=&'a Utf16Char>>(iter: I) -> Self {
- Self::from_iter(iter.into_iter().cloned())
- }
-}
-
-#[cfg(feature="std")]
-impl Extend<Utf16Char> for String {
- fn extend<I:IntoIterator<Item=Utf16Char>>(&mut self, iter: I) {
- self.extend(iter.into_iter().map(|u16c| Utf8Char::from(u16c) ));
- }
-}
-#[cfg(feature="std")]
-impl<'a> Extend<&'a Utf16Char> for String {
- fn extend<I:IntoIterator<Item=&'a Utf16Char>>(&mut self, iter: I) {
- self.extend(iter.into_iter().cloned());
- }
-}
-#[cfg(feature="std")]
-impl FromIterator<Utf16Char> for String {
- fn from_iter<I:IntoIterator<Item=Utf16Char>>(iter: I) -> Self {
- let mut s = String::new();
- s.extend(iter);
- return s;
- }
-}
-#[cfg(feature="std")]
-impl<'a> FromIterator<&'a Utf16Char> for String {
- fn from_iter<I:IntoIterator<Item=&'a Utf16Char>>(iter: I) -> Self {
- Self::from_iter(iter.into_iter().cloned())
- }
-}
-
-
- /////////////////
- //getter traits//
-/////////////////
-impl AsRef<[u16]> for Utf16Char {
- #[inline]
- fn as_ref(&self) -> &[u16] {
- &self.units[..self.len()]
- }
-}
-impl Borrow<[u16]> for Utf16Char {
- #[inline]
- fn borrow(&self) -> &[u16] {
- self.as_ref()
- }
-}
-impl Deref for Utf16Char {
- type Target = [u16];
- #[inline]
- fn deref(&self) -> &[u16] {
- self.as_ref()
- }
-}
-
-
- ////////////////
- //ascii traits//
-////////////////
-#[cfg(feature="std")]
-#[allow(deprecated)]
-impl AsciiExt for Utf16Char {
- type Owned = Self;
- fn is_ascii(&self) -> bool {
- self.units[0] < 128
- }
- fn eq_ignore_ascii_case(&self, other: &Self) -> bool {
- self.to_ascii_lowercase() == other.to_ascii_lowercase()
- }
- fn to_ascii_uppercase(&self) -> Self {
- let n = self.units[0].wrapping_sub(b'a' as u16);
- if n < 26 {Utf16Char{ units: [n+b'A' as u16, 0] }}
- else {*self}
- }
- fn to_ascii_lowercase(&self) -> Self {
- let n = self.units[0].wrapping_sub(b'A' as u16);
- if n < 26 {Utf16Char{ units: [n+b'a' as u16, 0] }}
- else {*self}
- }
- fn make_ascii_uppercase(&mut self) {
- *self = self.to_ascii_uppercase()
- }
- fn make_ascii_lowercase(&mut self) {
- *self = self.to_ascii_lowercase();
- }
-}
-
-#[cfg(feature="ascii")]
-/// Requires the feature "ascii".
-impl From<AsciiChar> for Utf16Char {
- #[inline]
- fn from(ac: AsciiChar) -> Self {
- Utf16Char{ units: [ac.as_byte() as u16, 0] }
- }
-}
-#[cfg(feature="ascii")]
-/// Requires the feature "ascii".
-impl ToAsciiChar for Utf16Char {
- #[inline]
- fn to_ascii_char(self) -> Result<AsciiChar, ToAsciiCharError> {
- // ToAsciiCHar is not implemented for u16 in ascii 0.9.0
- if self.is_ascii() {self.units[0] as u8} else {255}.to_ascii_char()
- }
- #[inline]
- unsafe fn to_ascii_char_unchecked(self) -> AsciiChar {
- (self.units[0] as u8).to_ascii_char_unchecked()
- }
-}
-
-
- /////////////////////////////////////////////////////////
- //Genaral traits that cannot be derived to emulate char//
-/////////////////////////////////////////////////////////
-impl hash::Hash for Utf16Char {
- fn hash<H : hash::Hasher>(&self, state: &mut H) {
- self.to_char().hash(state);
- }
-}
-impl fmt::Debug for Utf16Char {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- fmt::Debug::fmt(&self.to_char(), fmtr)
- }
-}
-impl fmt::Display for Utf16Char {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- fmt::Display::fmt(&Utf8Char::from(*self), fmtr)
- }
-}
-// Cannot derive these impls because two-unit characters must always compare
-// greater than one-unit ones.
-impl PartialOrd for Utf16Char {
- #[inline]
- fn partial_cmp(&self, rhs: &Self) -> Option<Ordering> {
- Some(self.cmp(rhs))
- }
-}
-impl Ord for Utf16Char {
- #[inline]
- fn cmp(&self, rhs: &Self) -> Ordering {
- // Shift the first unit by 0xd if surrogate, and 0 otherwise.
- // This ensures surrogates are always greater than 0xffff, and
- // that the second unit only affect the result when the first are equal.
- // Multiplying by a constant factor isn't enough because that factor
- // would have to be greater than 1023 and smaller than 5.5.
- // This transformation is less complicated than combine_surrogates().
- let lhs = (self.units[0] as u32, self.units[1] as u32);
- let rhs = (rhs.units[0] as u32, rhs.units[1] as u32);
- let lhs = (lhs.0 << (lhs.1 >> 12)) + lhs.1;
- let rhs = (rhs.0 << (rhs.1 >> 12)) + rhs.1;
- lhs.cmp(&rhs)
- }
-}
-
-
- ////////////////////////////////
- //Comparisons with other types//
-////////////////////////////////
-impl PartialEq<char> for Utf16Char {
- fn eq(&self, u32c: &char) -> bool {
- *self == Utf16Char::from(*u32c)
- }
-}
-impl PartialEq<Utf16Char> for char {
- fn eq(&self, u16c: &Utf16Char) -> bool {
- Utf16Char::from(*self) == *u16c
- }
-}
-impl PartialOrd<char> for Utf16Char {
- fn partial_cmp(&self, u32c: &char) -> Option<Ordering> {
- self.partial_cmp(&Utf16Char::from(*u32c))
- }
-}
-impl PartialOrd<Utf16Char> for char {
- fn partial_cmp(&self, u16c: &Utf16Char) -> Option<Ordering> {
- Utf16Char::from(*self).partial_cmp(u16c)
- }
-}
-
-impl PartialEq<Utf8Char> for Utf16Char {
- fn eq(&self, u8c: &Utf8Char) -> bool {
- *self == Utf16Char::from(*u8c)
- }
-}
-impl PartialOrd<Utf8Char> for Utf16Char {
- fn partial_cmp(&self, u8c: &Utf8Char) -> Option<Ordering> {
- self.partial_cmp(&Utf16Char::from(*u8c))
- }
-}
-// The other direction is implemented in utf8_char.rs
-
-/// Only considers the unit equal if the codepoint of the `Utf16Char` is not
-/// made up of a surrogate pair.
-///
-/// There is no impl in the opposite direction, as this should only be used to
-/// compare `Utf16Char`s against constants.
-///
-/// # Examples
-///
-/// ```
-/// # use encode_unicode::Utf16Char;
-/// assert!(Utf16Char::from('6') == b'6' as u16);
-/// assert!(Utf16Char::from('\u{FFFF}') == 0xffff_u16);
-/// assert!(Utf16Char::from_tuple((0xd876, Some(0xdef9))).unwrap() != 0xd876_u16);
-/// ```
-impl PartialEq<u16> for Utf16Char {
- fn eq(&self, unit: &u16) -> bool {
- self.units[0] == *unit && self.units[1] == 0
- }
-}
-/// Only considers the byte equal if the codepoint of the `Utf16Char` is <= U+FF.
-///
-/// # Examples
-///
-/// ```
-/// # use encode_unicode::Utf16Char;
-/// assert!(Utf16Char::from('6') == b'6');
-/// assert!(Utf16Char::from('\u{00FF}') == b'\xff');
-/// assert!(Utf16Char::from('\u{0100}') != b'\0');
-/// ```
-impl PartialEq<u8> for Utf16Char {
- fn eq(&self, byte: &u8) -> bool {
- self.units[0] == *byte as u16
- }
-}
-#[cfg(feature = "ascii")]
-/// `Utf16Char`s that are not ASCII never compare equal.
-impl PartialEq<AsciiChar> for Utf16Char {
- #[inline]
- fn eq(&self, ascii: &AsciiChar) -> bool {
- self.units[0] == *ascii as u16
- }
-}
-#[cfg(feature = "ascii")]
-/// `Utf16Char`s that are not ASCII never compare equal.
-impl PartialEq<Utf16Char> for AsciiChar {
- #[inline]
- fn eq(&self, u16c: &Utf16Char) -> bool {
- *self as u16 == u16c.units[0]
- }
-}
-#[cfg(feature = "ascii")]
-/// `Utf16Char`s that are not ASCII always compare greater.
-impl PartialOrd<AsciiChar> for Utf16Char {
- #[inline]
- fn partial_cmp(&self, ascii: &AsciiChar) -> Option<Ordering> {
- self.units[0].partial_cmp(&(*ascii as u16))
- }
-}
-#[cfg(feature = "ascii")]
-/// `Utf16Char`s that are not ASCII always compare greater.
-impl PartialOrd<Utf16Char> for AsciiChar {
- #[inline]
- fn partial_cmp(&self, u16c: &Utf16Char) -> Option<Ordering> {
- (*self as u16).partial_cmp(&u16c.units[0])
- }
-}
-
-
- ///////////////////////////////////////////////////////
- //pub impls that should be together for nicer rustdoc//
-///////////////////////////////////////////////////////
-impl Utf16Char {
- /// Create an `Utf16Char` from the first codepoint in a string slice,
- /// converting from UTF-8 to UTF-16.
- ///
- /// The returned `usize` is the number of UTF-8 bytes used from the str,
- /// and not the number of UTF-16 units.
- ///
- /// Returns an error if the `str` is empty.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::Utf16Char;
- ///
- /// assert_eq!(Utf16Char::from_str_start("a"), Ok((Utf16Char::from('a'),1)));
- /// assert_eq!(Utf16Char::from_str_start("ab"), Ok((Utf16Char::from('a'),1)));
- /// assert_eq!(Utf16Char::from_str_start("🂠 "), Ok((Utf16Char::from('🂠'),4)));
- /// assert_eq!(Utf16Char::from_str_start("é"), Ok((Utf16Char::from('e'),1)));// 'e'+u301 combining mark
- /// assert!(Utf16Char::from_str_start("").is_err());
- /// ```
- pub fn from_str_start(s: &str) -> Result<(Self,usize), EmptyStrError> {
- if s.is_empty() {
- return Err(EmptyStrError);
- }
- let b = s.as_bytes();
- // Read the last byte first to reduce the number of unnecesary length checks.
- match b[0] {
- 0...127 => {// 1 byte => 1 unit
- let unit = b[0] as u16;// 0b0000_0000_0xxx_xxxx
- Ok((Utf16Char{ units: [unit, 0] }, 1))
- },
- 0b1000_0000...0b1101_1111 => {// 2 bytes => 1 unit
- let unit = (((b[1] & 0x3f) as u16) << 0) // 0b0000_0000_00xx_xxxx
- | (((b[0] & 0x1f) as u16) << 6);// 0b0000_0xxx_xx00_0000
- Ok((Utf16Char{ units: [unit, 0] }, 2))
- },
- 0b1110_0000...0b1110_1111 => {// 3 bytes => 1 unit
- let unit = (((b[2] & 0x3f) as u16) << 0) // 0b0000_0000_00xx_xxxx
- | (((b[1] & 0x3f) as u16) << 6) // 0b0000_xxxx_xx00_0000
- | (((b[0] & 0x0f) as u16) << 12);// 0bxxxx_0000_0000_0000
- Ok((Utf16Char{ units: [unit, 0] }, 3))
- },
- _ => {// 4 bytes => 2 units
- let second = 0xdc00 // 0b1101_1100_0000_0000
- | (((b[3] & 0x3f) as u16) << 0) // 0b0000_0000_00xx_xxxx
- | (((b[2] & 0x0f) as u16) << 6);// 0b0000_00xx_xx00_0000
- let first = 0xd800-(0x01_00_00u32>>10) as u16// 0b1101_0111_1100_0000
- + (((b[2] & 0x30) as u16) >> 4) // 0b0000_0000_0000_00xx
- + (((b[1] & 0x3f) as u16) << 2) // 0b0000_0000_xxxx_xx00
- + (((b[0] & 0x07) as u16) << 8); // 0b0000_0xxx_0000_0000
- Ok((Utf16Char{ units: [first, second] }, 4))
- }
- }
- }
- /// Validate and store the first UTF-16 codepoint in the slice.
- /// Also return how many units were needed.
- pub fn from_slice_start(src: &[u16]) -> Result<(Self,usize), InvalidUtf16Slice> {
- char::from_utf16_slice_start(src).map(|(_,len)| {
- let second = if len==2 {src[1]} else {0};
- (Utf16Char{ units: [src[0], second] }, len)
- })
- }
- /// Store the first UTF-16 codepoint of the slice.
- ///
- /// # Safety
- ///
- /// The slice must be non-empty and start with a valid UTF-16 codepoint.
- /// The length of the slice is never checked.
- pub unsafe fn from_slice_start_unchecked(src: &[u16]) -> (Self,usize) {
- let first = *src.get_unchecked(0);
- if first.is_utf16_leading_surrogate() {
- (Utf16Char{ units: [first, *src.get_unchecked(1)] }, 2)
- } else {
- (Utf16Char{ units: [first, 0] }, 1)
- }
- }
- /// Validate and store an UTF-16 array as returned from `char.to_utf16_array()`.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::Utf16Char;
- /// use encode_unicode::error::InvalidUtf16Array;
- ///
- /// assert_eq!(Utf16Char::from_array(['x' as u16, 'y' as u16]), Ok(Utf16Char::from('x')));
- /// assert_eq!(Utf16Char::from_array(['睷' as u16, 0]), Ok(Utf16Char::from('睷')));
- /// assert_eq!(Utf16Char::from_array([0xda6f, 0xdcde]), Ok(Utf16Char::from('\u{abcde}')));
- /// assert_eq!(Utf16Char::from_array([0xf111, 0xdbad]), Ok(Utf16Char::from('\u{f111}')));
- /// assert_eq!(Utf16Char::from_array([0xdaaf, 0xdaaf]), Err(InvalidUtf16Array::SecondIsNotTrailingSurrogate));
- /// assert_eq!(Utf16Char::from_array([0xdcac, 0x9000]), Err(InvalidUtf16Array::FirstIsTrailingSurrogate));
- /// ```
- pub fn from_array(units: [u16; 2]) -> Result<Self,InvalidUtf16Array> {
- if (units[0] & 0xf8_00) != 0xd8_00 {
- Ok(Utf16Char { units: [units[0], 0] })
- } else if units[0] < 0xdc_00 && (units[1] & 0xfc_00) == 0xdc_00 {
- Ok(Utf16Char { units: units })
- } else if units[0] < 0xdc_00 {
- Err(InvalidUtf16Array::SecondIsNotTrailingSurrogate)
- } else {
- Err(InvalidUtf16Array::FirstIsTrailingSurrogate)
- }
- }
- /// Create an `Utf16Char` from an array as returned from `char.to_utf16_array()`.
- ///
- /// # Safety
- ///
- /// The units must form a valid codepoint, and the second unit must be 0
- /// when a surrogate pair is not required.
- /// Violating this can easily lead to undefined behavior, although unlike
- /// `char` bad `Utf16Char`s simply existing is not immediately UB.
- pub unsafe fn from_array_unchecked(units: [u16; 2]) -> Self {
- Utf16Char { units: units }
- }
- /// Validate and store a UTF-16 pair as returned from `char.to_utf16_tuple()`.
- pub fn from_tuple(utf16: (u16,Option<u16>)) -> Result<Self,InvalidUtf16Tuple> {
- unsafe {char::from_utf16_tuple(utf16).map(|_|
- Self::from_tuple_unchecked(utf16)
- )}
- }
- /// Create an `Utf16Char` from a tuple as returned from `char.to_utf16_tuple()`.
- ///
- /// # Safety
- ///
- /// The units must form a valid codepoint with the second being 0 when a
- /// surrogate pair is not required.
- /// Violating this can easily lead to undefined behavior.
- pub unsafe fn from_tuple_unchecked(utf16: (u16,Option<u16>)) -> Self {
- Utf16Char { units: [utf16.0, utf16.1.unwrap_or(0)] }
- }
- /// Create an `Utf16Char` from a single unit.
- ///
- /// Codepoints < '\u{1_0000}' (which fit in a `u16`) are part of the basic
- /// multilingual plane unless they are reserved for surrogate pairs.
- ///
- /// # Errors
- ///
- /// Returns `NonBMPError` if the unit is in the range `0xd800..0xe000`
- /// (which means that it's part of a surrogat pair)
- ///
- /// # Examples
- ///
- /// ```
- /// # use encode_unicode::Utf16Char;
- /// assert_eq!(Utf16Char::from_bmp(0x40).unwrap(), '@');
- /// assert_eq!(Utf16Char::from_bmp('ø' as u16).unwrap(), 'ø');
- /// assert!(Utf16Char::from_bmp(0xdddd).is_err());
- /// ```
- pub fn from_bmp(bmp_codepoint: u16) -> Result<Self,NonBMPError> {
- if bmp_codepoint & 0xf800 != 0xd800 {
- Ok(Utf16Char{ units: [bmp_codepoint, 0] })
- } else {
- Err(NonBMPError)
- }
- }
- /// Create an `Utf16Char` from a single unit without checking that it's a
- /// valid codepoint on its own.
- ///
- /// # Safety
- ///
- /// The unit must be less than 0xd800 or greater than 0xdfff.
- /// In other words, not part of a surrogate pair.
- /// Violating this can easily lead to undefined behavior.
- #[inline]
- pub unsafe fn from_bmp_unchecked(bmp_codepoint: u16) -> Self {
- Utf16Char{ units: [bmp_codepoint, 0] }
- }
- /// Checks that the codepoint is in the basic multilingual plane.
- ///
- /// # Examples
- /// ```
- /// # use encode_unicode::Utf16Char;
- /// assert_eq!(Utf16Char::from('e').is_bmp(), true);
- /// assert_eq!(Utf16Char::from('€').is_bmp(), true);
- /// assert_eq!(Utf16Char::from('𝔼').is_bmp(), false);
- /// ```
- #[inline]
- pub fn is_bmp(&self) -> bool {
- self.units[1] == 0
- }
-
- /// The number of units this character is made up of.
- ///
- /// Is either 1 or 2 and identical to `.as_char().len_utf16()`
- /// or `.as_ref().len()`.
- #[inline]
- pub fn len(self) -> usize {
- 1 + (self.units[1] as usize >> 15)
- }
- // There is no `.is_emty()` because it would always return false.
-
- /// Checks that the codepoint is an ASCII character.
- #[inline]
- pub fn is_ascii(&self) -> bool {
- self.units[0] <= 127
- }
- /// Checks that two characters are an ASCII case-insensitive match.
- ///
- /// Is equivalent to `a.to_ascii_lowercase() == b.to_ascii_lowercase()`.
- #[cfg(feature="std")]
- pub fn eq_ignore_ascii_case(&self, other: &Self) -> bool {
- self.to_ascii_lowercase() == other.to_ascii_lowercase()
- }
- /// Converts the character to its ASCII upper case equivalent.
- ///
- /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z',
- /// but non-ASCII letters are unchanged.
- #[cfg(feature="std")]
- pub fn to_ascii_uppercase(&self) -> Self {
- let n = self.units[0].wrapping_sub(b'a' as u16);
- if n < 26 {Utf16Char{ units: [n+b'A' as u16, 0] }}
- else {*self}
- }
- /// Converts the character to its ASCII lower case equivalent.
- ///
- /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z',
- /// but non-ASCII letters are unchanged.
- #[cfg(feature="std")]
- pub fn to_ascii_lowercase(&self) -> Self {
- let n = self.units[0].wrapping_sub(b'A' as u16);
- if n < 26 {Utf16Char{ units: [n+b'a' as u16, 0] }}
- else {*self}
- }
- /// Converts the character to its ASCII upper case equivalent in-place.
- ///
- /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z',
- /// but non-ASCII letters are unchanged.
- #[cfg(feature="std")]
- pub fn make_ascii_uppercase(&mut self) {
- *self = self.to_ascii_uppercase()
- }
- /// Converts the character to its ASCII lower case equivalent in-place.
- ///
- /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z',
- /// but non-ASCII letters are unchanged.
- #[cfg(feature="std")]
- pub fn make_ascii_lowercase(&mut self) {
- *self = self.to_ascii_lowercase();
- }
-
- /// Convert from UTF-16 to UTF-32
- pub fn to_char(self) -> char {
- self.into()
- }
- /// Write the internal representation to a slice,
- /// and then returns the number of `u16`s written.
- ///
- /// # Panics
- /// Will panic the buffer is too small;
- /// You can get the required length from `.len()`,
- /// but a buffer of length two is always large enough.
- pub fn to_slice(self, dst: &mut[u16]) -> usize {
- // Write the last unit first to avoid repeated length checks.
- let extra = self.units[1] as usize >> 15;
- match dst.get_mut(extra) {
- Some(first) => *first = self.units[extra],
- None => panic!("The provided buffer is too small.")
- }
- if extra != 0 {dst[0] = self.units[0];}
- extra+1
- }
- /// Get the character represented as an array of two units.
- ///
- /// The second `u16` is zero for codepoints that fit in one unit.
- #[inline]
- pub fn to_array(self) -> [u16;2] {
- self.units
- }
- /// The second `u16` is used for surrogate pairs.
- #[inline]
- pub fn to_tuple(self) -> (u16,Option<u16>) {
- (self.units[0], if self.units[1]==0 {None} else {Some(self.units[1])})
- }
-}
diff --git a/vendor/encode_unicode/src/utf16_iterators.rs b/vendor/encode_unicode/src/utf16_iterators.rs
deleted file mode 100644
index 7adb5ac..0000000
--- a/vendor/encode_unicode/src/utf16_iterators.rs
+++ /dev/null
@@ -1,270 +0,0 @@
-/* Copyright 2016 The encode_unicode Developers
- *
- * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
- * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
- * http://opensource.org/licenses/MIT>, at your option. This file may not be
- * copied, modified, or distributed except according to those terms.
- */
-
-use traits::CharExt;
-use utf16_char::Utf16Char;
-use errors::EmptyStrError;
-extern crate core;
-use self::core::fmt;
-use self::core::borrow::Borrow;
-
-// Invalid values that says the field is consumed or empty.
-const FIRST_USED: u16 = 0x_dc_00;
-const SECOND_USED: u16 = 0;
-
-/// Iterate over the units of the UTF-16 representation of a codepoint.
-#[derive(Clone)]
-pub struct Utf16Iterator {
- first: u16,
- second: u16,
-}
-impl From<char> for Utf16Iterator {
- fn from(c: char) -> Self {
- let (first, second) = c.to_utf16_tuple();
- Utf16Iterator{ first: first, second: second.unwrap_or(SECOND_USED) }
- }
-}
-impl From<Utf16Char> for Utf16Iterator {
- fn from(uc: Utf16Char) -> Self {
- let (first, second) = uc.to_tuple();
- Utf16Iterator{ first: first, second: second.unwrap_or(SECOND_USED) }
- }
-}
-impl Iterator for Utf16Iterator {
- type Item=u16;
- fn next(&mut self) -> Option<u16> {
- match (self.first, self.second) {
- (FIRST_USED, SECOND_USED) => { None },
- (FIRST_USED, second ) => {self.second = SECOND_USED; Some(second)},
- (first , _ ) => {self.first = FIRST_USED; Some(first )},
- }
- }
- fn size_hint(&self) -> (usize, Option<usize>) {
- (self.len(), Some(self.len()))
- }
-}
-impl ExactSizeIterator for Utf16Iterator {
- fn len(&self) -> usize {
- (if self.first == FIRST_USED {0} else {1}) +
- (if self.second == SECOND_USED {0} else {1})
- }
-}
-impl fmt::Debug for Utf16Iterator {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- let mut clone = self.clone();
- match (clone.next(), clone.next()) {
- (Some(one), None) => write!(fmtr, "[{}]", one),
- (Some(a), Some(b)) => write!(fmtr, "[{}, {}]", a, b),
- (None, _) => write!(fmtr, "[]"),
- }
- }
-}
-
-
-
-/// Converts an iterator of `Utf16Char` (or `&Utf16Char`)
-/// to an iterator of `u16`s.
-/// Is equivalent to calling `.flat_map()` on the original iterator,
-/// but the returned iterator is about twice as fast.
-///
-/// The exact number of units cannot be known in advance, but `size_hint()`
-/// gives the possible range.
-///
-/// # Examples
-///
-/// From iterator of values:
-///
-/// ```
-/// use encode_unicode::{iter_units, CharExt};
-///
-/// let iterator = "foo".chars().map(|c| c.to_utf16() );
-/// let mut units = [0; 4];
-/// for (u,dst) in iter_units(iterator).zip(&mut units) {*dst=u;}
-/// assert_eq!(units, ['f' as u16, 'o' as u16, 'o' as u16, 0]);
-/// ```
-///
-/// From iterator of references:
-///
-#[cfg_attr(feature="std", doc=" ```")]
-#[cfg_attr(not(feature="std"), doc=" ```no_compile")]
-/// use encode_unicode::{iter_units, CharExt, Utf16Char};
-///
-/// // (💣 takes two units)
-/// let chars: Vec<Utf16Char> = "💣 bomb 💣".chars().map(|c| c.to_utf16() ).collect();
-/// let units: Vec<u16> = iter_units(&chars).collect();
-/// let flat_map: Vec<u16> = chars.iter().flat_map(|u16c| *u16c ).collect();
-/// assert_eq!(units, flat_map);
-/// ```
-pub fn iter_units<U:Borrow<Utf16Char>, I:IntoIterator<Item=U>>
-(iterable: I) -> Utf16CharSplitter<U, I::IntoIter> {
- Utf16CharSplitter{ inner: iterable.into_iter(), prev_second: 0 }
-}
-
-/// The iterator type returned by `iter_units()`
-#[derive(Clone)]
-pub struct Utf16CharSplitter<U:Borrow<Utf16Char>, I:Iterator<Item=U>> {
- inner: I,
- prev_second: u16,
-}
-impl<I:Iterator<Item=Utf16Char>> From<I> for Utf16CharSplitter<Utf16Char,I> {
- /// A less generic constructor than `iter_units()`
- fn from(iter: I) -> Self {
- iter_units(iter)
- }
-}
-impl<U:Borrow<Utf16Char>, I:Iterator<Item=U>> Utf16CharSplitter<U,I> {
- /// Extracts the source iterator.
- ///
- /// Note that `iter_units(iter.into_inner())` is not a no-op:
- /// If the last returned unit from `next()` was a leading surrogate,
- /// the trailing surrogate is lost.
- pub fn into_inner(self) -> I {
- self.inner
- }
-}
-impl<U:Borrow<Utf16Char>, I:Iterator<Item=U>> Iterator for Utf16CharSplitter<U,I> {
- type Item = u16;
- fn next(&mut self) -> Option<Self::Item> {
- if self.prev_second == 0 {
- self.inner.next().map(|u16c| {
- let units = u16c.borrow().to_array();
- self.prev_second = units[1];
- units[0]
- })
- } else {
- let prev_second = self.prev_second;
- self.prev_second = 0;
- Some(prev_second)
- }
- }
- fn size_hint(&self) -> (usize,Option<usize>) {
- // Doesn't need to handle unlikely overflows correctly because
- // size_hint() cannot be relied upon anyway. (the trait isn't unsafe)
- let (min, max) = self.inner.size_hint();
- let add = if self.prev_second == 0 {0} else {1};
- (min.wrapping_add(add), max.map(|max| max.wrapping_mul(2).wrapping_add(add) ))
- }
-}
-
-
-
-/// An iterator over the codepoints in a `str` represented as `Utf16Char`.
-#[derive(Clone)]
-pub struct Utf16CharIndices<'a>{
- str: &'a str,
- index: usize,
-}
-impl<'a> From<&'a str> for Utf16CharIndices<'a> {
- fn from(s: &str) -> Utf16CharIndices {
- Utf16CharIndices{str: s, index: 0}
- }
-}
-impl<'a> Utf16CharIndices<'a> {
- /// Extract the remainder of the source `str`.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::{StrExt, Utf16Char};
- /// let mut iter = "abc".utf16char_indices();
- /// assert_eq!(iter.next_back(), Some((2, Utf16Char::from('c'))));
- /// assert_eq!(iter.next(), Some((0, Utf16Char::from('a'))));
- /// assert_eq!(iter.as_str(), "b");
- /// ```
- pub fn as_str(&self) -> &'a str {
- &self.str[self.index..]
- }
-}
-impl<'a> Iterator for Utf16CharIndices<'a> {
- type Item = (usize,Utf16Char);
- fn next(&mut self) -> Option<(usize,Utf16Char)> {
- match Utf16Char::from_str_start(&self.str[self.index..]) {
- Ok((u16c, bytes)) => {
- let item = (self.index, u16c);
- self.index += bytes;
- Some(item)
- },
- Err(EmptyStrError) => None
- }
- }
- fn size_hint(&self) -> (usize,Option<usize>) {
- let len = self.str.len() - self.index;
- // For len+3 to overflow, the slice must fill all but two bytes of
- // addressable memory, and size_hint() doesn't need to be correct.
- (len.wrapping_add(3)/4, Some(len))
- }
-}
-impl<'a> DoubleEndedIterator for Utf16CharIndices<'a> {
- fn next_back(&mut self) -> Option<(usize,Utf16Char)> {
- if self.index < self.str.len() {
- let rev = self.str.bytes().rev();
- let len = 1 + rev.take_while(|b| b & 0b1100_0000 == 0b1000_0000 ).count();
- let starts = self.str.len() - len;
- let (u16c,_) = Utf16Char::from_str_start(&self.str[starts..]).unwrap();
- self.str = &self.str[..starts];
- Some((starts, u16c))
- } else {
- None
- }
- }
-}
-impl<'a> fmt::Debug for Utf16CharIndices<'a> {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- fmtr.debug_tuple("Utf16CharIndices")
- .field(&self.index)
- .field(&self.as_str())
- .finish()
- }
-}
-
-
-/// An iterator over the codepoints in a `str` represented as `Utf16Char`.
-#[derive(Clone)]
-pub struct Utf16Chars<'a>(Utf16CharIndices<'a>);
-impl<'a> From<&'a str> for Utf16Chars<'a> {
- fn from(s: &str) -> Utf16Chars {
- Utf16Chars(Utf16CharIndices::from(s))
- }
-}
-impl<'a> Utf16Chars<'a> {
- /// Extract the remainder of the source `str`.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::{StrExt, Utf16Char};
- /// let mut iter = "abc".utf16chars();
- /// assert_eq!(iter.next(), Some(Utf16Char::from('a')));
- /// assert_eq!(iter.next_back(), Some(Utf16Char::from('c')));
- /// assert_eq!(iter.as_str(), "b");
- /// ```
- pub fn as_str(&self) -> &'a str {
- self.0.as_str()
- }
-}
-impl<'a> Iterator for Utf16Chars<'a> {
- type Item = Utf16Char;
- fn next(&mut self) -> Option<Utf16Char> {
- self.0.next().map(|(_,u16c)| u16c )
- }
- fn size_hint(&self) -> (usize,Option<usize>) {
- self.0.size_hint()
- }
-}
-impl<'a> DoubleEndedIterator for Utf16Chars<'a> {
- fn next_back(&mut self) -> Option<Utf16Char> {
- self.0.next_back().map(|(_,u16c)| u16c )
- }
-}
-impl<'a> fmt::Debug for Utf16Chars<'a> {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- fmtr.debug_tuple("Utf16Chars")
- .field(&self.as_str())
- .finish()
- }
-}
diff --git a/vendor/encode_unicode/src/utf8_char.rs b/vendor/encode_unicode/src/utf8_char.rs
deleted file mode 100644
index 9dba4ba..0000000
--- a/vendor/encode_unicode/src/utf8_char.rs
+++ /dev/null
@@ -1,647 +0,0 @@
-/* Copyright 2016 The encode_unicode Developers
- *
- * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
- * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
- * http://opensource.org/licenses/MIT>, at your option. This file may not be
- * copied, modified, or distributed except according to those terms.
- */
-
-use errors::{FromStrError, EmptyStrError, NonAsciiError, InvalidUtf8Slice, InvalidUtf8Array};
-use utf8_iterators::Utf8Iterator;
-use traits::{CharExt, U8UtfExt};
-use utf16_char::Utf16Char;
-extern crate core;
-use self::core::{hash, fmt, str, ptr};
-use self::core::cmp::Ordering;
-use self::core::borrow::Borrow;
-use self::core::ops::Deref;
-use self::core::mem::transmute;
-#[cfg(feature="std")]
-use self::core::iter::FromIterator;
-#[cfg(feature="std")]
-#[allow(deprecated)]
-use std::ascii::AsciiExt;
-#[cfg(feature="ascii")]
-extern crate ascii;
-#[cfg(feature="ascii")]
-use self::ascii::{AsciiChar,ToAsciiChar,ToAsciiCharError};
-
-
-// I don't think there is any good default value for char, but char does.
-#[derive(Default)]
-// char doesn't do anything more advanced than u32 for Eq/Ord, so we shouldn't either.
-// The default impl of Ord for arrays works out because longer codepoints
-// start with more ones, so if they're equal, the length is the same,
-// breaks down for values above 0x1f_ff_ff but those can only be created by unsafe code.
-#[derive(PartialEq,Eq, PartialOrd,Ord)]
-
-#[derive(Clone,Copy)]
-
-
-/// An unicode codepoint stored as UTF-8.
-///
-/// It can be borrowed as a `str`, and has the same size as `char`.
-pub struct Utf8Char {
- bytes: [u8; 4],
-}
-
-
- /////////////////////
- //conversion traits//
-/////////////////////
-impl str::FromStr for Utf8Char {
- type Err = FromStrError;
- /// Create an `Utf8Char` from a string slice.
- /// The string must contain exactly one codepoint.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::error::FromStrError::*;
- /// use encode_unicode::Utf8Char;
- /// use std::str::FromStr;
- ///
- /// assert_eq!(Utf8Char::from_str("a"), Ok(Utf8Char::from('a')));
- /// assert_eq!(Utf8Char::from_str("🂠"), Ok(Utf8Char::from('🂠')));
- /// assert_eq!(Utf8Char::from_str(""), Err(Empty));
- /// assert_eq!(Utf8Char::from_str("ab"), Err(MultipleCodepoints));
- /// assert_eq!(Utf8Char::from_str("é"), Err(MultipleCodepoints));// 'e'+u301 combining mark
- /// ```
- fn from_str(s: &str) -> Result<Self, FromStrError> {
- if s.is_empty() {
- Err(FromStrError::Empty)
- } else if s.len() != 1+s.as_bytes()[0].extra_utf8_bytes_unchecked() {
- Err(FromStrError::MultipleCodepoints)
- } else {
- let mut bytes = [0; 4];
- bytes[..s.len()].copy_from_slice(s.as_bytes());
- Ok(Utf8Char{bytes: bytes})
- }
- }
-}
-impl From<Utf16Char> for Utf8Char {
- fn from(utf16: Utf16Char) -> Utf8Char {
- match utf16.to_tuple() {
- (a @ 0...0x00_7f, _) => {
- Utf8Char{ bytes: [a as u8, 0, 0, 0] }
- },
- (u @ 0...0x07_ff, _) => {
- let b = 0x80 | (u & 0x00_3f) as u8;
- let a = 0xc0 | ((u & 0x07_c0) >> 6) as u8;
- Utf8Char{ bytes: [a, b, 0, 0] }
- },
- (u, None) => {
- let c = 0x80 | (u & 0x00_3f) as u8;
- let b = 0x80 | ((u & 0x0f_c0) >> 6) as u8;
- let a = 0xe0 | ((u & 0xf0_00) >> 12) as u8;
- Utf8Char{ bytes: [a, b, c, 0] }
- },
- (f, Some(s)) => {
- let f = f + (0x01_00_00u32 >> 10) as u16;
- let d = 0x80 | (s & 0x00_3f) as u8;
- let c = 0x80 | ((s & 0x03_c0) >> 6) as u8
- | ((f & 0x00_03) << 4) as u8;
- let b = 0x80 | ((f & 0x00_fc) >> 2) as u8;
- let a = 0xf0 | ((f & 0x07_00) >> 8) as u8;
- Utf8Char{ bytes: [a, b, c, d] }
- }
- }
- }
-}
-impl From<char> for Utf8Char {
- fn from(c: char) -> Self {
- Utf8Char{ bytes: c.to_utf8_array().0 }
- }
-}
-impl From<Utf8Char> for char {
- fn from(uc: Utf8Char) -> char {
- unsafe{ char::from_utf8_exact_slice_unchecked(&uc.bytes[..uc.len()]) }
- }
-}
-impl IntoIterator for Utf8Char {
- type Item=u8;
- type IntoIter=Utf8Iterator;
- /// Iterate over the byte values.
- fn into_iter(self) -> Utf8Iterator {
- Utf8Iterator::from(self)
- }
-}
-
-#[cfg(feature="std")]
-impl Extend<Utf8Char> for Vec<u8> {
- fn extend<I:IntoIterator<Item=Utf8Char>>(&mut self, iter: I) {
- let iter = iter.into_iter();
- self.reserve(iter.size_hint().0);
- for u8c in iter {
- // twice as fast as self.extend_from_slice(u8c.as_bytes());
- self.push(u8c.bytes[0]);
- for &extra in &u8c.bytes[1..] {
- if extra != 0 {
- self.push(extra);
- }
- }
- }
- }
-}
-#[cfg(feature="std")]
-impl<'a> Extend<&'a Utf8Char> for Vec<u8> {
- fn extend<I:IntoIterator<Item=&'a Utf8Char>>(&mut self, iter: I) {
- self.extend(iter.into_iter().cloned())
- }
-}
-#[cfg(feature="std")]
-impl Extend<Utf8Char> for String {
- fn extend<I:IntoIterator<Item=Utf8Char>>(&mut self, iter: I) {
- unsafe { self.as_mut_vec().extend(iter) }
- }
-}
-#[cfg(feature="std")]
-impl<'a> Extend<&'a Utf8Char> for String {
- fn extend<I:IntoIterator<Item=&'a Utf8Char>>(&mut self, iter: I) {
- self.extend(iter.into_iter().cloned())
- }
-}
-#[cfg(feature="std")]
-impl FromIterator<Utf8Char> for String {
- fn from_iter<I:IntoIterator<Item=Utf8Char>>(iter: I) -> String {
- let mut string = String::new();
- string.extend(iter);
- return string;
- }
-}
-#[cfg(feature="std")]
-impl<'a> FromIterator<&'a Utf8Char> for String {
- fn from_iter<I:IntoIterator<Item=&'a Utf8Char>>(iter: I) -> String {
- iter.into_iter().cloned().collect()
- }
-}
-#[cfg(feature="std")]
-impl FromIterator<Utf8Char> for Vec<u8> {
- fn from_iter<I:IntoIterator<Item=Utf8Char>>(iter: I) -> Self {
- iter.into_iter().collect::<String>().into_bytes()
- }
-}
-#[cfg(feature="std")]
-impl<'a> FromIterator<&'a Utf8Char> for Vec<u8> {
- fn from_iter<I:IntoIterator<Item=&'a Utf8Char>>(iter: I) -> Self {
- iter.into_iter().cloned().collect::<String>().into_bytes()
- }
-}
-
-
- /////////////////
- //getter traits//
-/////////////////
-impl AsRef<[u8]> for Utf8Char {
- fn as_ref(&self) -> &[u8] {
- &self.bytes[..self.len()]
- }
-}
-impl AsRef<str> for Utf8Char {
- fn as_ref(&self) -> &str {
- unsafe{ str::from_utf8_unchecked( self.as_ref() ) }
- }
-}
-impl Borrow<[u8]> for Utf8Char {
- fn borrow(&self) -> &[u8] {
- self.as_ref()
- }
-}
-impl Borrow<str> for Utf8Char {
- fn borrow(&self) -> &str {
- self.as_ref()
- }
-}
-impl Deref for Utf8Char {
- type Target = str;
- fn deref(&self) -> &Self::Target {
- self.as_ref()
- }
-}
-
-
- ////////////////
- //ascii traits//
-////////////////
-#[cfg(feature="std")]
-#[allow(deprecated)]
-impl AsciiExt for Utf8Char {
- type Owned = Utf8Char;
- fn is_ascii(&self) -> bool {
- self.bytes[0].is_ascii()
- }
- fn eq_ignore_ascii_case(&self, other: &Self) -> bool {
- if self.is_ascii() {self.bytes[0].eq_ignore_ascii_case(&other.bytes[0])}
- else {self == other}
- }
- fn to_ascii_uppercase(&self) -> Self::Owned {
- let mut uc = *self;
- uc.make_ascii_uppercase();
- uc
- }
- fn to_ascii_lowercase(&self) -> Self::Owned {
- let mut uc = *self;
- uc.make_ascii_lowercase();
- uc
- }
- fn make_ascii_uppercase(&mut self) {
- self.bytes[0].make_ascii_uppercase()
- }
- fn make_ascii_lowercase(&mut self) {
- self.bytes[0].make_ascii_lowercase();
- }
-}
-
-#[cfg(feature="ascii")]
-/// Requires the feature "ascii".
-impl From<AsciiChar> for Utf8Char {
- fn from(ac: AsciiChar) -> Self {
- Utf8Char{ bytes: [ac.as_byte(),0,0,0] }
- }
-}
-#[cfg(feature="ascii")]
-/// Requires the feature "ascii".
-impl ToAsciiChar for Utf8Char {
- fn to_ascii_char(self) -> Result<AsciiChar, ToAsciiCharError> {
- self.bytes[0].to_ascii_char()
- }
- unsafe fn to_ascii_char_unchecked(self) -> AsciiChar {
- self.bytes[0].to_ascii_char_unchecked()
- }
-}
-
-
- /////////////////////////////////////////////////////////
- //Genaral traits that cannot be derived to emulate char//
-/////////////////////////////////////////////////////////
-impl hash::Hash for Utf8Char {
- fn hash<H : hash::Hasher>(&self, state: &mut H) {
- self.to_char().hash(state);
- }
-}
-impl fmt::Debug for Utf8Char {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- fmt::Debug::fmt(&self.to_char(), fmtr)
- }
-}
-impl fmt::Display for Utf8Char {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- fmtr.write_str(self.as_str())
- }
-}
-
-
- ////////////////////////////////
- //Comparisons with other types//
-////////////////////////////////
-impl PartialEq<char> for Utf8Char {
- fn eq(&self, u32c: &char) -> bool {
- *self == Utf8Char::from(*u32c)
- }
-}
-impl PartialEq<Utf8Char> for char {
- fn eq(&self, u8c: &Utf8Char) -> bool {
- Utf8Char::from(*self) == *u8c
- }
-}
-impl PartialOrd<char> for Utf8Char {
- fn partial_cmp(&self, u32c: &char) -> Option<Ordering> {
- self.partial_cmp(&Self::from(*u32c))
- }
-}
-impl PartialOrd<Utf8Char> for char {
- fn partial_cmp(&self, u8c: &Utf8Char) -> Option<Ordering> {
- Utf8Char::from(*self).partial_cmp(u8c)
- }
-}
-
-impl PartialEq<Utf16Char> for Utf8Char {
- fn eq(&self, u16c: &Utf16Char) -> bool {
- *self == Self::from(*u16c)
- }
-}
-impl PartialOrd<Utf16Char> for Utf8Char {
- fn partial_cmp(&self, u16c: &Utf16Char) -> Option<Ordering> {
- self.partial_cmp(&Self::from(*u16c))
- }
-}
-// The other direction is implemented in utf16_char.rs
-
-/// Only considers the byte equal if both it and the `Utf8Char` represents ASCII characters.
-///
-/// There is no impl in the opposite direction, as this should only be used to
-/// compare `Utf8Char`s against constants.
-///
-/// # Examples
-///
-/// ```
-/// # use encode_unicode::Utf8Char;
-/// assert!(Utf8Char::from('8') == b'8');
-/// assert!(Utf8Char::from_array([0xf1,0x80,0x80,0x80]).unwrap() != 0xf1);
-/// assert!(Utf8Char::from('\u{ff}') != 0xff);
-/// assert!(Utf8Char::from('\u{80}') != 0x80);
-/// ```
-impl PartialEq<u8> for Utf8Char {
- fn eq(&self, byte: &u8) -> bool {
- self.bytes[0] == *byte && self.bytes[1] == 0
- }
-}
-#[cfg(feature = "ascii")]
-/// `Utf8Char`s that are not ASCII never compare equal.
-impl PartialEq<AsciiChar> for Utf8Char {
- #[inline]
- fn eq(&self, ascii: &AsciiChar) -> bool {
- self.bytes[0] == *ascii as u8
- }
-}
-#[cfg(feature = "ascii")]
-/// `Utf8Char`s that are not ASCII never compare equal.
-impl PartialEq<Utf8Char> for AsciiChar {
- #[inline]
- fn eq(&self, u8c: &Utf8Char) -> bool {
- u8c == self
- }
-}
-#[cfg(feature = "ascii")]
-/// `Utf8Char`s that are not ASCII always compare greater.
-impl PartialOrd<AsciiChar> for Utf8Char {
- #[inline]
- fn partial_cmp(&self, ascii: &AsciiChar) -> Option<Ordering> {
- self.bytes[0].partial_cmp(ascii)
- }
-}
-#[cfg(feature = "ascii")]
-/// `Utf8Char`s that are not ASCII always compare greater.
-impl PartialOrd<Utf8Char> for AsciiChar {
- #[inline]
- fn partial_cmp(&self, u8c: &Utf8Char) -> Option<Ordering> {
- self.partial_cmp(&u8c.bytes[0])
- }
-}
-
-
- ///////////////////////////////////////////////////////
- //pub impls that should be together for nicer rustdoc//
-///////////////////////////////////////////////////////
-impl Utf8Char {
- /// Create an `Utf8Char` from the first codepoint in a `str`.
- ///
- /// Returns an error if the `str` is empty.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::Utf8Char;
- ///
- /// assert_eq!(Utf8Char::from_str_start("a"), Ok((Utf8Char::from('a'),1)));
- /// assert_eq!(Utf8Char::from_str_start("ab"), Ok((Utf8Char::from('a'),1)));
- /// assert_eq!(Utf8Char::from_str_start("🂠 "), Ok((Utf8Char::from('🂠'),4)));
- /// assert_eq!(Utf8Char::from_str_start("é"), Ok((Utf8Char::from('e'),1)));// 'e'+u301 combining mark
- /// assert!(Utf8Char::from_str_start("").is_err());
- /// ```
- pub fn from_str_start(src: &str) -> Result<(Self,usize),EmptyStrError> {
- unsafe {
- if src.is_empty() {
- Err(EmptyStrError)
- } else {
- Ok(Utf8Char::from_slice_start_unchecked(src.as_bytes()))
- }
- }
- }
- /// Create an `Utf8Char` of the first codepoint in an UTF-8 slice.
- /// Also returns the length of the UTF-8 sequence for the codepoint.
- ///
- /// If the slice is from a `str`, use `::from_str_start()` to skip UTF-8 validation.
- ///
- /// # Errors
- ///
- /// Returns an `Err` if the slice is empty, doesn't start with a valid
- /// UTF-8 sequence or is too short for the sequence.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::Utf8Char;
- /// use encode_unicode::error::InvalidUtf8Slice::*;
- /// use encode_unicode::error::InvalidUtf8::*;
- ///
- /// assert_eq!(Utf8Char::from_slice_start(&[b'A', b'B', b'C']), Ok((Utf8Char::from('A'),1)));
- /// assert_eq!(Utf8Char::from_slice_start(&[0xdd, 0xbb]), Ok((Utf8Char::from('\u{77b}'),2)));
- ///
- /// assert_eq!(Utf8Char::from_slice_start(&[]), Err(TooShort(1)));
- /// assert_eq!(Utf8Char::from_slice_start(&[0xf0, 0x99]), Err(TooShort(4)));
- /// assert_eq!(Utf8Char::from_slice_start(&[0xee, b'F', 0x80]), Err(Utf8(NotAContinuationByte(1))));
- /// assert_eq!(Utf8Char::from_slice_start(&[0xee, 0x99, 0x0f]), Err(Utf8(NotAContinuationByte(2))));
- /// ```
- pub fn from_slice_start(src: &[u8]) -> Result<(Self,usize),InvalidUtf8Slice> {
- char::from_utf8_slice_start(src).map(|(_,len)| {
- let mut bytes = [0; 4];
- bytes[..len].copy_from_slice(&src[..len]);
- (Utf8Char{ bytes: bytes }, len)
- })
- }
- /// A `from_slice_start()` that doesn't validate the codepoint.
- ///
- /// # Safety
- ///
- /// The slice must be non-empty and start with a valid UTF-8 codepoint.
- /// Invalid or incomplete values might cause reads of uninitalized memory.
- pub unsafe fn from_slice_start_unchecked(src: &[u8]) -> (Self,usize) {
- let len = 1+src.get_unchecked(0).extra_utf8_bytes_unchecked();
- let mut bytes = [0; 4];
- ptr::copy_nonoverlapping(src.as_ptr(), &mut bytes[0] as *mut u8, len);
- (Utf8Char{ bytes: bytes }, len)
- }
- /// Create an `Utf8Char` from a byte array after validating it.
- ///
- /// The codepoint must start at the first byte.
- /// Unused bytes are set to zero by this function and so can be anything.
- ///
- /// # Errors
- ///
- /// Returns an `Err` if the array doesn't start with a valid UTF-8 sequence.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::Utf8Char;
- /// use encode_unicode::error::InvalidUtf8Array::*;
- /// use encode_unicode::error::InvalidUtf8::*;
- /// use encode_unicode::error::InvalidCodepoint::*;
- ///
- /// assert_eq!(Utf8Char::from_array([b'A', 0, 0, 0]), Ok(Utf8Char::from('A')));
- /// assert_eq!(Utf8Char::from_array([0xf4, 0x8b, 0xbb, 0xbb]), Ok(Utf8Char::from('\u{10befb}')));
- /// assert_eq!(Utf8Char::from_array([b'A', b'B', b'C', b'D']), Ok(Utf8Char::from('A')));
- /// assert_eq!(Utf8Char::from_array([0, 0, 0xcc, 0xbb]), Ok(Utf8Char::from('\0')));
- ///
- /// assert_eq!(Utf8Char::from_array([0xef, b'F', 0x80, 0x80]), Err(Utf8(NotAContinuationByte(1))));
- /// assert_eq!(Utf8Char::from_array([0xc1, 0x80, 0, 0]), Err(Utf8(OverLong)));
- /// assert_eq!(Utf8Char::from_array([0xf7, 0xaa, 0x99, 0x88]), Err(Codepoint(TooHigh)));
- /// ```
- pub fn from_array(utf8: [u8;4]) -> Result<Self,InvalidUtf8Array> {
- unsafe {
- // perform all validation
- try!(char::from_utf8_array(utf8));
- let extra = utf8[0].extra_utf8_bytes_unchecked() as u32;
- // zero unused bytes in one operation by transmuting the arrary to
- // u32, apply an endian-corrected mask and transmute back
- let mask = u32::from_le(0xff_ff_ff_ff >> 8*(3-extra));
- let unused_zeroed = mask & transmute::<_,u32>(utf8);
- Ok(Utf8Char{ bytes: transmute(unused_zeroed) })
- }
- }
- /// Zero-cost constructor.
- ///
- /// # Safety
- ///
- /// Must contain a valid codepoint starting at the first byte, with the
- /// unused bytes zeroed.
- /// Bad values can easily lead to undefined behavior.
- #[inline]
- pub unsafe fn from_array_unchecked(utf8: [u8;4]) -> Self {
- Utf8Char{ bytes: utf8 }
- }
- /// Create an `Utf8Char` from a single byte.
- ///
- /// The byte must be an ASCII character.
- ///
- /// # Errors
- ///
- /// Returns `NonAsciiError` if the byte greater than 127.
- ///
- /// # Examples
- ///
- /// ```
- /// # use encode_unicode::Utf8Char;
- /// assert_eq!(Utf8Char::from_ascii(b'a').unwrap(), 'a');
- /// assert!(Utf8Char::from_ascii(128).is_err());
- /// ```
- pub fn from_ascii(ascii: u8) -> Result<Self,NonAsciiError> {
- if ascii as i8 >= 0 {
- Ok(Utf8Char{ bytes: [ascii, 0, 0, 0] })
- } else {
- Err(NonAsciiError)
- }
- }
- /// Create an `Utf8Char` from a single byte without checking that it's a
- /// valid codepoint on its own, which is only true for ASCII characters.
- ///
- /// # Safety
- ///
- /// The byte must be less than 128.
- #[inline]
- pub unsafe fn from_ascii_unchecked(ascii: u8) -> Self {
- Utf8Char{ bytes: [ascii, 0, 0, 0] }
- }
-
- /// The number of bytes this character needs.
- ///
- /// Is between 1 and 4 (inclusive) and identical to `.as_ref().len()` or
- /// `.as_char().len_utf8()`.
- #[inline]
- pub fn len(self) -> usize {
- // Invariants of the extra bytes enambles algorithms that
- // `u8.extra_utf8_bytes_unchecked()` cannot use.
- // Some of them turned out to require fewer x86 instructions:
-
- // Exploits that unused bytes are zero and calculates the number of
- // trailing zero bytes.
- // Setting a bit in the first byte prevents the function from returning
- // 0 for '\0' (which has 32 leading zeros).
- // trailing and leading is swapped below to optimize for little-endian
- // architectures.
- (4 - (u32::to_le(unsafe{transmute(self.bytes)})|1).leading_zeros()/8) as usize
-
- // Exploits that the extra bytes have their most significant bit set if
- // in use.
- // Takes fewer instructions than the one above if popcnt can be used,
- // (which it cannot by default,
- // set RUSTFLAGS='-C target-cpu=native' to enable)
- //let all: u32 = unsafe{transmute(self.bytes)};
- //let msb_mask = u32::from_be(0x00808080);
- //let add_one = u32::from_be(0x80000000);
- //((all & msb_mask) | add_one).count_ones() as usize
- }
- // There is no .is_emty() because this type is never empty.
-
- /// Checks that the codepoint is an ASCII character.
- pub fn is_ascii(&self) -> bool {
- self.bytes[0] <= 127
- }
- /// Checks that two characters are an ASCII case-insensitive match.
- ///
- /// Is equivalent to `a.to_ascii_lowercase() == b.to_ascii_lowercase()`.
- #[cfg(feature="std")]
- pub fn eq_ignore_ascii_case(&self, other: &Self) -> bool {
- if self.is_ascii() {self.bytes[0].eq_ignore_ascii_case(&other.bytes[0])}
- else {self == other}
- }
- /// Converts the character to its ASCII upper case equivalent.
- ///
- /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z',
- /// but non-ASCII letters are unchanged.
- #[cfg(feature="std")]
- pub fn to_ascii_uppercase(&self) -> Self {
- let mut uc = *self;
- uc.make_ascii_uppercase();
- uc
- }
- /// Converts the character to its ASCII lower case equivalent.
- ///
- /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z',
- /// but non-ASCII letters are unchanged.
- #[cfg(feature="std")]
- pub fn to_ascii_lowercase(&self) -> Self {
- let mut uc = *self;
- uc.make_ascii_lowercase();
- uc
- }
- /// Converts the character to its ASCII upper case equivalent in-place.
- ///
- /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z',
- /// but non-ASCII letters are unchanged.
- #[inline]
- #[cfg(feature="std")]
- pub fn make_ascii_uppercase(&mut self) {
- self.bytes[0].make_ascii_uppercase()
- }
- /// Converts the character to its ASCII lower case equivalent in-place.
- ///
- /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z',
- /// but non-ASCII letters are unchanged.
- #[inline]
- #[cfg(feature="std")]
- pub fn make_ascii_lowercase(&mut self) {
- self.bytes[0].make_ascii_lowercase();
- }
-
- /// Convert from UTF-8 to UTF-32
- pub fn to_char(self) -> char {
- self.into()
- }
- /// Write the internal representation to a slice,
- /// and then returns the number of bytes written.
- ///
- /// # Panics
- ///
- /// Will panic the buffer is too small;
- /// You can get the required length from `.len()`,
- /// but a buffer of length four is always large enough.
- pub fn to_slice(self, dst: &mut[u8]) -> usize {
- if self.len() > dst.len() {
- panic!("The provided buffer is too small.");
- }
- dst[..self.len()].copy_from_slice(&self.bytes[..self.len()]);
- self.len()
- }
- /// Expose the internal array and the number of used bytes.
- pub fn to_array(self) -> ([u8;4],usize) {
- (self.bytes, self.len())
- }
- /// Return a `str` view of the array the codepoint is stored as.
- ///
- /// Is an unambiguous version of `.as_ref()`.
- pub fn as_str(&self) -> &str {
- self.deref()
- }
-}
diff --git a/vendor/encode_unicode/src/utf8_iterators.rs b/vendor/encode_unicode/src/utf8_iterators.rs
deleted file mode 100644
index 891d729..0000000
--- a/vendor/encode_unicode/src/utf8_iterators.rs
+++ /dev/null
@@ -1,352 +0,0 @@
-/* Copyright 2016 The encode_unicode Developers
- *
- * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
- * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
- * http://opensource.org/licenses/MIT>, at your option. This file may not be
- * copied, modified, or distributed except according to those terms.
- */
-
-use utf8_char::Utf8Char;
-use errors::EmptyStrError;
-extern crate core;
-use self::core::{mem, u32, u64};
-use self::core::ops::Not;
-use self::core::fmt;
-use self::core::borrow::Borrow;
-#[cfg(feature="std")]
-use std::io::{Read, Error as ioError};
-
-
-
-/// Read or iterate over the bytes of the UTF-8 representation of a codepoint.
-#[derive(Clone)]
-pub struct Utf8Iterator (u32);
-
-impl From<Utf8Char> for Utf8Iterator {
- fn from(uc: Utf8Char) -> Self {
- let used = u32::from_le(unsafe{ mem::transmute(uc.to_array().0) });
- // uses u64 because shifting an u32 by 32 bits is a no-op.
- let unused_set = (u64::MAX << uc.len() as u64*8) as u32;
- Utf8Iterator(used | unused_set)
- }
-}
-impl From<char> for Utf8Iterator {
- fn from(c: char) -> Self {
- Self::from(Utf8Char::from(c))
- }
-}
-impl Iterator for Utf8Iterator {
- type Item=u8;
- fn next(&mut self) -> Option<u8> {
- let next = self.0 as u8;
- if next == 0xff {
- None
- } else {
- self.0 = (self.0 >> 8) | 0xff_00_00_00;
- Some(next)
- }
- }
- fn size_hint(&self) -> (usize, Option<usize>) {
- (self.len(), Some(self.len()))
- }
-}
-impl ExactSizeIterator for Utf8Iterator {
- fn len(&self) -> usize {// not straightforward, but possible
- let unused_bytes = self.0.not().leading_zeros() / 8;
- 4 - unused_bytes as usize
- }
-}
-#[cfg(feature="std")]
-impl Read for Utf8Iterator {
- /// Always returns Ok
- fn read(&mut self, buf: &mut[u8]) -> Result<usize, ioError> {
- // Cannot call self.next() until I know I can write the result.
- for (i, dst) in buf.iter_mut().enumerate() {
- match self.next() {
- Some(b) => *dst = b,
- None => return Ok(i),
- }
- }
- Ok(buf.len())
- }
-}
-impl fmt::Debug for Utf8Iterator {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- let mut content = [0; 4];
- let mut i = 0;
- for b in self.clone() {
- content[i] = b;
- i += 1;
- }
- write!(fmtr, "{:?}", &content[..i])
- }
-}
-
-
-
-/// Converts an iterator of `Utf8Char` (or `&Utf8Char`)
-/// to an iterator of `u8`s.
-/// Is equivalent to calling `.flat_map()` on the original iterator,
-/// but the returned iterator is ~40% faster.
-///
-/// The iterator also implements `Read` (if the `std` feature isn't disabled).
-/// Reading will never produce an error, and calls to `.read()` and `.next()`
-/// can be mixed.
-///
-/// The exact number of bytes cannot be known in advance, but `size_hint()`
-/// gives the possible range.
-/// (min: all remaining characters are ASCII, max: all require four bytes)
-///
-/// # Examples
-///
-/// From iterator of values:
-///
-/// ```
-/// use encode_unicode::{iter_bytes, CharExt};
-///
-/// let iterator = "foo".chars().map(|c| c.to_utf8() );
-/// let mut bytes = [0; 4];
-/// for (u,dst) in iter_bytes(iterator).zip(&mut bytes) {*dst=u;}
-/// assert_eq!(&bytes, b"foo\0");
-/// ```
-///
-/// From iterator of references:
-///
-#[cfg_attr(feature="std", doc=" ```")]
-#[cfg_attr(not(feature="std"), doc=" ```no_compile")]
-/// use encode_unicode::{iter_bytes, CharExt, Utf8Char};
-///
-/// let chars: Vec<Utf8Char> = "💣 bomb 💣".chars().map(|c| c.to_utf8() ).collect();
-/// let bytes: Vec<u8> = iter_bytes(&chars).collect();
-/// let flat_map: Vec<u8> = chars.iter().flat_map(|u8c| *u8c ).collect();
-/// assert_eq!(bytes, flat_map);
-/// ```
-///
-/// `Read`ing from it:
-///
-#[cfg_attr(feature="std", doc=" ```")]
-#[cfg_attr(not(feature="std"), doc=" ```no_compile")]
-/// use encode_unicode::{iter_bytes, CharExt};
-/// use std::io::Read;
-///
-/// let s = "Ååh‽";
-/// assert_eq!(s.len(), 8);
-/// let mut buf = [b'E'; 9];
-/// let mut reader = iter_bytes(s.chars().map(|c| c.to_utf8() ));
-/// assert_eq!(reader.read(&mut buf[..]).unwrap(), 8);
-/// assert_eq!(reader.read(&mut buf[..]).unwrap(), 0);
-/// assert_eq!(&buf[..8], s.as_bytes());
-/// assert_eq!(buf[8], b'E');
-/// ```
-pub fn iter_bytes<U:Borrow<Utf8Char>, I:IntoIterator<Item=U>>
-(iterable: I) -> Utf8CharSplitter<U, I::IntoIter> {
- Utf8CharSplitter{ inner: iterable.into_iter(), prev: 0 }
-}
-
-/// The iterator type returned by `iter_bytes()`
-///
-/// See its documentation for details.
-#[derive(Clone)]
-pub struct Utf8CharSplitter<U:Borrow<Utf8Char>, I:Iterator<Item=U>> {
- inner: I,
- prev: u32,
-}
-impl<I:Iterator<Item=Utf8Char>> From<I> for Utf8CharSplitter<Utf8Char,I> {
- /// A less generic constructor than `iter_bytes()`
- fn from(iter: I) -> Self {
- iter_bytes(iter)
- }
-}
-impl<U:Borrow<Utf8Char>, I:Iterator<Item=U>> Utf8CharSplitter<U,I> {
- /// Extracts the source iterator.
- ///
- /// Note that `iter_bytes(iter.into_inner())` is not a no-op:
- /// If the last returned byte from `next()` was not an ASCII by,
- /// the remaining bytes of that codepoint is lost.
- pub fn into_inner(self) -> I {
- self.inner
- }
-}
-impl<U:Borrow<Utf8Char>, I:Iterator<Item=U>> Iterator for Utf8CharSplitter<U,I> {
- type Item = u8;
- fn next(&mut self) -> Option<Self::Item> {
- if self.prev == 0 {
- self.inner.next().map(|u8c| {
- let array = u8c.borrow().to_array().0;
- self.prev = unsafe{ u32::from_le(mem::transmute(array)) } >> 8;
- array[0]
- })
- } else {
- let next = self.prev as u8;
- self.prev >>= 8;
- Some(next)
- }
- }
- fn size_hint(&self) -> (usize,Option<usize>) {
- // Doesn't need to handle unlikely overflows correctly because
- // size_hint() cannot be relied upon anyway. (the trait isn't unsafe)
- let (min, max) = self.inner.size_hint();
- let add = 4 - (self.prev.leading_zeros() / 8) as usize;
- (min.wrapping_add(add), max.map(|max| max.wrapping_mul(4).wrapping_add(add) ))
- }
-}
-#[cfg(feature="std")]
-impl<U:Borrow<Utf8Char>, I:Iterator<Item=U>> Read for Utf8CharSplitter<U,I> {
- /// Always returns `Ok`
- fn read(&mut self, buf: &mut[u8]) -> Result<usize, ioError> {
- let mut i = 0;
- // write remaining bytes of previous codepoint
- while self.prev != 0 && i < buf.len() {
- buf[i] = self.prev as u8;
- self.prev >>= 8;
- i += 1;
- }
- // write whole characters
- while i < buf.len() {
- let bytes = match self.inner.next() {
- Some(u8c) => u8c.borrow().to_array().0,
- None => break
- };
- buf[i] = bytes[0];
- i += 1;
- if bytes[1] != 0 {
- let len = bytes[0].not().leading_zeros() as usize;
- let mut written = 1;
- while written < len {
- if i < buf.len() {
- buf[i] = bytes[written];
- i += 1;
- written += 1;
- } else {
- let bytes_as_u32 = unsafe{ u32::from_le(mem::transmute(bytes)) };
- self.prev = bytes_as_u32 >> (8*written);
- return Ok(i);
- }
- }
- }
- }
- Ok(i)
- }
-}
-
-
-
-/// An iterator over the `Utf8Char` of a string slice, and their positions.
-///
-/// This struct is created by the `utf8char_indices() method from [`StrExt`] trait. See its documentation for more.
-#[derive(Clone)]
-pub struct Utf8CharIndices<'a>{
- str: &'a str,
- index: usize,
-}
-impl<'a> From<&'a str> for Utf8CharIndices<'a> {
- fn from(s: &str) -> Utf8CharIndices {
- Utf8CharIndices{str: s, index: 0}
- }
-}
-impl<'a> Utf8CharIndices<'a> {
- /// Extract the remainder of the source `str`.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::{StrExt, Utf8Char};
- /// let mut iter = "abc".utf8char_indices();
- /// assert_eq!(iter.next_back(), Some((2, Utf8Char::from('c'))));
- /// assert_eq!(iter.next(), Some((0, Utf8Char::from('a'))));
- /// assert_eq!(iter.as_str(), "b");
- /// ```
- pub fn as_str(&self) -> &'a str {
- &self.str[self.index..]
- }
-}
-impl<'a> Iterator for Utf8CharIndices<'a> {
- type Item = (usize,Utf8Char);
- fn next(&mut self) -> Option<(usize,Utf8Char)> {
- match Utf8Char::from_str_start(&self.str[self.index..]) {
- Ok((u8c, len)) => {
- let item = (self.index, u8c);
- self.index += len;
- Some(item)
- },
- Err(EmptyStrError) => None
- }
- }
- fn size_hint(&self) -> (usize,Option<usize>) {
- let len = self.str.len() - self.index;
- // For len+3 to overflow, the slice must fill all but two bytes of
- // addressable memory, and size_hint() doesn't need to be correct.
- (len.wrapping_add(3)/4, Some(len))
- }
-}
-impl<'a> DoubleEndedIterator for Utf8CharIndices<'a> {
- fn next_back(&mut self) -> Option<(usize,Utf8Char)> {
- // Cannot refactor out the unwrap without switching to ::from_slice()
- // since slicing the str panics if not on a boundary.
- if self.index < self.str.len() {
- let rev = self.str.bytes().rev();
- let len = 1 + rev.take_while(|b| b & 0b1100_0000 == 0b1000_0000 ).count();
- let starts = self.str.len() - len;
- let (u8c,_) = Utf8Char::from_str_start(&self.str[starts..]).unwrap();
- self.str = &self.str[..starts];
- Some((starts, u8c))
- } else {
- None
- }
- }
-}
-impl<'a> fmt::Debug for Utf8CharIndices<'a> {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- fmtr.debug_tuple("Utf8CharIndices")
- .field(&self.index)
- .field(&self.as_str())
- .finish()
- }
-}
-
-
-/// An iterator over the codepoints in a `str` represented as `Utf8Char`.
-#[derive(Clone)]
-pub struct Utf8Chars<'a>(Utf8CharIndices<'a>);
-impl<'a> From<&'a str> for Utf8Chars<'a> {
- fn from(s: &str) -> Utf8Chars {
- Utf8Chars(Utf8CharIndices::from(s))
- }
-}
-impl<'a> Utf8Chars<'a> {
- /// Extract the remainder of the source `str`.
- ///
- /// # Examples
- ///
- /// ```
- /// use encode_unicode::{StrExt, Utf8Char};
- /// let mut iter = "abc".utf8chars();
- /// assert_eq!(iter.next(), Some(Utf8Char::from('a')));
- /// assert_eq!(iter.next_back(), Some(Utf8Char::from('c')));
- /// assert_eq!(iter.as_str(), "b");
- /// ```
- pub fn as_str(&self) -> &'a str {
- self.0.as_str()
- }
-}
-impl<'a> Iterator for Utf8Chars<'a> {
- type Item = Utf8Char;
- fn next(&mut self) -> Option<Utf8Char> {
- self.0.next().map(|(_,u8c)| u8c )
- }
- fn size_hint(&self) -> (usize,Option<usize>) {
- self.0.size_hint()
- }
-}
-impl<'a> DoubleEndedIterator for Utf8Chars<'a> {
- fn next_back(&mut self) -> Option<Utf8Char> {
- self.0.next_back().map(|(_,u8c)| u8c )
- }
-}
-impl<'a> fmt::Debug for Utf8Chars<'a> {
- fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result {
- fmtr.debug_tuple("Utf8CharIndices")
- .field(&self.as_str())
- .finish()
- }
-}
Copyright © 2024 Valentin A. Popov. All Rights Reverse Engineered.