diff options
Diffstat (limited to 'vendor/encode_unicode')
20 files changed, 0 insertions, 5017 deletions
diff --git a/vendor/encode_unicode/.cargo-checksum.json b/vendor/encode_unicode/.cargo-checksum.json deleted file mode 100644 index 75bdcff..0000000 --- a/vendor/encode_unicode/.cargo-checksum.json +++ /dev/null @@ -1 +0,0 @@ -{"files":{"AUTHORS.md":"f2cf336738ad935a482a799be004083ddd07c904513caf80f9e48011888fe1b6","Cargo.toml":"6fd14a963bfb44b78883bb57c082300138668bc163c949286453836116c7018d","LICENSE-APACHE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","LICENSE-MIT":"da23be69ad3ccf7a5823d62152efcd2b15de9482a5014fcb1b02844662d86abd","README.md":"c33df8cbe2645cd55d3644d4af453c0bd3cf3ffaa4a0c15ac6f3162fade966d6","RELEASES.md":"0a10f449adcf53ab00a43bb0242e38b508a51b529b3afc02eb645dbb0216b3bc","benches/multiiterators.rs":"69c878d010856a24247085356ed3045c6ceb1ac88cd75ea7a00b11206090debe","src/decoding_iterators.rs":"72c9fe0d10240e021dfc46546e814f04ab5c7e142a04305fad8c8da48575fe92","src/errors.rs":"0355e926edd1c8e81b537aca1a80fc324912a8c21e84db278c232860e3476822","src/lib.rs":"972010cd7f1b24dd048d066f1a3ff57fc16d4486a7e3583f7ae995dbd1ada5c8","src/traits.rs":"7ec1b649f23410e55bbfe6df13713040bcd292ee90a81d31437291ad100ea99f","src/utf16_char.rs":"c014de07ebc08592b3527e62d66699c8d637c8c9491835341ccdc71e28f346a1","src/utf16_iterators.rs":"9344132fb95077f05b6da8d9da77eb38ddfc8134e543313d74f286e7e545e875","src/utf8_char.rs":"349a3ebafa8ae2c88efa334958ea4d1863ca559d1b17aeb2c53113cd9afc7d15","src/utf8_iterators.rs":"e3d3bbb23253a582c48985b4eb26a4febfb40fc79a12e5fc1ca28c4cdef9fe81","tests/errs.rs":"7244966b93fc98c19a9ca163870863c1fba6107d526f8c2baa065696eb4cf9b4","tests/exhaustive.rs":"25c71761e57ac45125c2d527ddc5fc5e7b89b9b1055df3db50bce81af9844250","tests/iterators.rs":"1bda1ea031950134eef6b21e8473f4c7a2d338822cd40bb47267db4fe608d586","tests/oks.rs":"9c3e571488bc66696f7cd518f89e25bbb0ad382cbf2834f7461043237c42f6d9"},"package":"a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"}
\ No newline at end of file diff --git a/vendor/encode_unicode/AUTHORS.md b/vendor/encode_unicode/AUTHORS.md deleted file mode 100644 index 6759605..0000000 --- a/vendor/encode_unicode/AUTHORS.md +++ /dev/null @@ -1,4 +0,0 @@ -# The encode_unicode Developers - -* Torbjørn Birch Moltu -* Aljoscha Meyer diff --git a/vendor/encode_unicode/Cargo.toml b/vendor/encode_unicode/Cargo.toml deleted file mode 100644 index 618c9c7..0000000 --- a/vendor/encode_unicode/Cargo.toml +++ /dev/null @@ -1,39 +0,0 @@ -# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO -# -# When uploading crates to the registry Cargo will automatically -# "normalize" Cargo.toml files for maximal compatibility -# with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies -# -# If you believe there's an error in this file please file an -# issue against the rust-lang/cargo repository. If you're -# editing this file be aware that the upstream Cargo.toml -# will likely look very different (and much more reasonable) - -[package] -name = "encode_unicode" -version = "0.3.6" -authors = ["Torbjørn Birch Moltu <t.b.moltu@lyse.net>"] -description = "UTF-8 and UTF-16 character types, iterators and related methods for char, u8 and u16.\n" -documentation = "https://docs.rs/encode_unicode/" -readme = "README.md" -keywords = ["unicode", "UTF-8", "UTF-16"] -categories = ["encoding", "no-std"] -license = "MIT/Apache-2.0" -repository = "https://github.com/tormol/encode_unicode" -[package.metadata.docs.rs] -features = ["ascii/std"] -[dependencies.ascii] -version = ">=0.8, <2" -optional = true -default-features = false - -[dependencies.clippy] -version = "0.*" -optional = true - -[features] -default = ["std"] -std = [] -[target."cfg(unix)".dev-dependencies.lazy_static] -version = "1.0.*" diff --git a/vendor/encode_unicode/LICENSE-APACHE b/vendor/encode_unicode/LICENSE-APACHE deleted file mode 100644 index d645695..0000000 --- a/vendor/encode_unicode/LICENSE-APACHE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/encode_unicode/LICENSE-MIT b/vendor/encode_unicode/LICENSE-MIT deleted file mode 100644 index de22e42..0000000 --- a/vendor/encode_unicode/LICENSE-MIT +++ /dev/null @@ -1,17 +0,0 @@ -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE diff --git a/vendor/encode_unicode/README.md b/vendor/encode_unicode/README.md deleted file mode 100644 index b1bfc6c..0000000 --- a/vendor/encode_unicode/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# encode_unicode - -UTF-8 and UTF-16 character types, iterators and related methods for `char`, `u8` and `u16`. - -[](https://crates.io/crates/encode_unicode/) - -## Features - -* **[`Utf8Char`](https://docs.rs/encode_unicode/latest/encode_unicode/struct.Utf8Char.html)**: - A `char` stored as UTF-8. Can be borrowed as a `str` or `u8` slice. -* **[`Utf16Char`](https://docs.rs/encode_unicode/latest/encode_unicode/struct.Utf16Char.html)**: - A `char` stored as UTF-16. Can be borrowed as an `u16` slice. -* [Conversion methods on `char`](https://docs.rs/encode_unicode/latest/encode_unicode/trait.CharExt.html): - * to and from UTF-8 as `[u8; 4]` or slice. - * to and from UTF-16 as `(u16, Option<u16>)` or slice. -* [Iterator adapters](https://docs.rs/encode_unicode/latest/encode_unicode/trait.IterExt.html) - for converting betwenn `u8`s and `Utf8Char`s or `u16`s and `Utf16Char`s. -* Optimized [slice-based decoding iterators](https://docs.rs/encode_unicode/latest/encode_unicode/trait.SliceExt.html). -* [Precise errors when decoding a char from UTF-8, UTF-16 or `u32` fails](http://docs.rs/encode_unicode/latest/encode_unicode/error/index.html). -* Utility methods on [`u8`](https://docs.rs/encode_unicode/latest/encode_unicode/trait.U8UtfExt.html) - and [`u16`](https://docs.rs/encode_unicode/latest/encode_unicode/trait.U16UtfExt.html). - -The minimum supported version of Rust is 1.15, -older versions might work now but can break with a minor update. - -## Optional features - -* `#![no_std]`-mode: There are a few differences: - * `Error` doesn't exist, but `description()` is made available as an inherent impl. - * `Extend`/`FromIterator`-implementations for `String`/`Vec<u8>`/`Vec<u16>` are missing. - * There is no `io`, so `Utf8Iterator` and `Utf8CharSplitter` doesn't implement `Read`. - This feature is enabled by setting `default-features=false` in `Cargo.toml`: - `encode_unicode = {version="0.3.4", default-features=false}`. -* Integration with the [ascii](https://tomprogrammer.github.io/rust-ascii/ascii/index.html) crate: - Convert `Utf8Char` and `Utf16Char` to and from [ascii::`AsciiChar`](https://tomprogrammer.github.io/rust-ascii/ascii/enum.AsciiChar.html). - -## License - -Licensed under either of - -* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) -* MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) - -at your option. - -### Contribution - -Unless you explicitly state otherwise, any contribution intentionally -submitted for inclusion in the work by you, as defined in the Apache-2.0 -license, shall be dual licensed as above, without any additional terms or -conditions. - -## History - -The original purpose of this crate was to provide standins for the then -unstable `encode_utf8()` and `encode_utf16()`. -The standins were removed in 0.3 when Rust 1.15 stabilized the `encode_` -methods, but the other stuff I added, such as iterators like -those `encode_utf{8,16}() returned for a while, might still be of use. diff --git a/vendor/encode_unicode/RELEASES.md b/vendor/encode_unicode/RELEASES.md deleted file mode 100644 index 371f8bf..0000000 --- a/vendor/encode_unicode/RELEASES.md +++ /dev/null @@ -1,74 +0,0 @@ -Version 0.3.6 (2019-08-23) -========================== -* Fix pointless undefined behavior in `Utf16Char.to_ascii_char()` (which is part of ascii feature) -* Widen ascii version requirement to include 1.* -* Add `[u16; 2]` UTF-16 array alternatives to `(u16, Some(u16))` UTF-16 tuple methods -* Add `Utf16Char.is_bmp()` - -Version 0.3.5 (2018-10-23) -========================== -* Fix docs.rs build failure - -Version 0.3.4 (2018-10-23) -========================== -* Fix UB in UTF-8 validation which lead to invalid codepoints being accepted in release mode -* Add fallible decoding iterator adapters `Utf8CharMerger` and `Utf16CharMerger` - and slice-based iterators `Utf8CharDecoder` and `Utf16CharDecoder` -* Widen ascii version requirement from 0.8.* to 0.8.0 - 0.10.* -* Implement creating / extending `String`s from `Utf16Char`-producing iterators - -Version 0.3.3 (2018-10-16) -========================== -* Fix UTF-8 overlong check. (`from_array()` and `from_slice()` accepted two-byte encodings of ASCII characters >= '@', which includes all letters) -* Implement `FromStr` for `Utf16Char` -* Add `from_str_start()` to `Utf8Char` and `Utf16Char` -* Add `Utf{8,16}Char{s,Indices}`: `str`-based iterators for `Utf8Char` and `Utf16Char` equivalent to `char`'s `Chars` and `CharIndices`. -* Add `StrExt` with functions to create the above iterators. -* Implement `FromIterator` and `Extend` for `Vec<{u8,u16}>` with reference-producing `Utf{8,16}Char` iterators too. -* Add `Utf8CharSplitter` and `Utf16CharSplitter`: `Utf{8,16}Char`-to-`u{8,16}` iterator adapters. -* Add `IterExt`, `iter_bytes()` and `iter_units()` to create the above splitting iterators. -* Add `Utf8Char::from_ascii()`, `Utf16Char::from_bmp()` with `_unchecked` versions of both. -* Add cross-type `PartialEq` and `PartialOrd` implementations. -* Change the `description()` for a few error types. - -Version 0.3.2 (2018-08-08) -========================== -* Hide `AsciiExt` deprecation warning and add replacement methods. -* Correct documentation for `U8UtfExt::extra_utf8_bytes()`. -* Fix misspellings in some error descriptions. -* Avoid potentially bad transmutes. - -Version 0.3.1 (2017-06-16) -========================== -* Implement `Display` for `Utf8Char` and `Utf16Char`. - -Version 0.3.0 (2017-03-29) -========================== -* Replace the "no_std" feature with opt-out "std". - * Upgrade ascii to v0.8. - * Make tests compile on stable. -* Remove `CharExt::write_utf{8,16}()` because `encode_utf{8,16}()` has been stabilized. -* Return a proper error from `U16UtfExt::utf16_needs_extra_unit()` instead of `None`. -* Rename `U16UtfExt::utf_is_leading_surrogate()` to `is_utf16_leading_surrogate()`. -* Rename `Utf16Char::from_slice()` to `from_slice_start()` and `CharExt::from_utf{8,16}_slice()` - to `from_utf{8,16}_slice_start()` to be consistent with `Utf8Char`. -* Fix a bug where `CharExt::from_slice()` would accept some trailing surrogates - as standalone codepoints. - -Version 0.2.0 (2016-07-24) -========================== -* Change `CharExt::write_utf{8,16}()` to panic instead of returning `None` - if the slice is too short. -* Fix bug where `CharExt::write_utf8()` and `Utf8Char::to_slice()` could change bytes it shouldn't. -* Rename lots of errors with search and replace: - * CodePoint -> Codepoint - * Several -> Multiple -* Update the ascii feature to use [ascii](https://tomprogrammer.github.io/rust-ascii/ascii/index.html) v0.7. -* Support `#[no_std]`; see 70e090ee for differences. -* Ungate impls of `AsciiExt`. (doesn't require ascii or nightly) -* Make the tests compile (and pass) again. - (They still require nightly). - -Version 0.1.* (2016-04-07) -========================== -First release. diff --git a/vendor/encode_unicode/benches/multiiterators.rs b/vendor/encode_unicode/benches/multiiterators.rs deleted file mode 100644 index 22c3eed..0000000 --- a/vendor/encode_unicode/benches/multiiterators.rs +++ /dev/null @@ -1,93 +0,0 @@ -// uses /usr/share/dict/ for text to convert to Vec<Utf*Char> and iterate over -#![cfg(all(unix, feature="std"))] -#![feature(test)] -extern crate test; -use test::{Bencher, black_box}; -#[macro_use] extern crate lazy_static; -extern crate encode_unicode; -use encode_unicode::{CharExt, Utf8Char, Utf16Char, iter_bytes, iter_units}; - -static ENGLISH: &str = include_str!("/usr/share/dict/american-english"); -// TODO find a big chinese file; `aptitude search '?provides(wordlist)'` didn't have one -lazy_static!{ - static ref UTF8CHARS: Vec<Utf8Char> = ENGLISH.chars().map(|c| c.to_utf8() ).collect(); - static ref UTF16CHARS: Vec<Utf16Char> = ENGLISH.chars().map(|c| c.to_utf16() ).collect(); -} - - -#[bench] -fn utf16_split_all_single_mulititerator(b: &mut Bencher) { - b.iter(|| { - iter_units(black_box(&*UTF16CHARS)).for_each(|u| assert!(u != 0) ); - }); -} -#[bench] -fn utf16_split_all_single_flatmap(b: &mut Bencher) { - b.iter(|| { - black_box(&*UTF16CHARS).iter().flat_map(|&u16c| u16c ).for_each(|u| assert!(u != 0) ); - }); -} -#[bench] -fn utf16_split_all_single_cloned_flatten(b: &mut Bencher) { - b.iter(|| { - black_box(&*UTF16CHARS).iter().cloned().flatten().for_each(|u| assert!(u != 0) ); - }); -} - - -#[bench] -fn utf8_split_mostly_ascii_multiiterator(b: &mut Bencher) { - b.iter(|| { - iter_bytes(black_box(&*UTF8CHARS)).for_each(|b| assert!(b != 0) ); - }); -} -#[bench] -fn utf8_split_mostly_ascii_flatmap(b: &mut Bencher) { - b.iter(|| { - black_box(&*UTF8CHARS).iter().flat_map(|&u8c| u8c ).for_each(|b| assert!(b != 0) ); - }); -} -#[bench] -fn utf8_split_mostly_ascii_cloned_flatten(b: &mut Bencher) { - b.iter(|| { - black_box(&*UTF8CHARS).iter().cloned().flatten().for_each(|b| assert!(b != 0) ); - }); -} - - -#[bench] -fn utf8_extend_mostly_ascii_multiiterator(b: &mut Bencher) { - b.iter(|| { - let vec: Vec<u8> = iter_bytes(black_box(&*UTF8CHARS)).collect(); - assert_eq!(black_box(vec).len(), ENGLISH.len()); - }); -} -#[bench] -fn utf8_extend_mostly_ascii_custom(b: &mut Bencher) { - b.iter(|| { - let vec: Vec<u8> = black_box(&*UTF8CHARS).iter().collect(); - assert_eq!(black_box(vec).len(), ENGLISH.len()); - }); -} -#[bench] -fn utf8_extend_mostly_ascii_custom_str(b: &mut Bencher) { - b.iter(|| { - let vec: String = black_box(&*UTF8CHARS).iter().cloned().collect(); - assert_eq!(black_box(vec).len(), ENGLISH.len()); - }); -} - -#[bench] -fn utf16_extend_all_single_multiiterator(b: &mut Bencher) { - b.iter(|| { - let vec: Vec<u16> = iter_units(black_box(&*UTF16CHARS)).collect(); - assert!(black_box(vec).len() < ENGLISH.len()); - }); -} -#[bench] -fn utf16_extend_all_single_custom(b: &mut Bencher) { - b.iter(|| { - let vec: Vec<u16> = black_box(&*UTF16CHARS).iter().collect(); - assert!(black_box(vec).len() < ENGLISH.len()); - }); -} diff --git a/vendor/encode_unicode/src/decoding_iterators.rs b/vendor/encode_unicode/src/decoding_iterators.rs deleted file mode 100644 index 4ef4125..0000000 --- a/vendor/encode_unicode/src/decoding_iterators.rs +++ /dev/null @@ -1,494 +0,0 @@ -/* Copyright 2018 The encode_unicode Developers - * - * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or - * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or - * http://opensource.org/licenses/MIT>, at your option. This file may not be - * copied, modified, or distributed except according to those terms. - */ - -//! Iterators that turn multiple `u8`s or `u16`s into `Utf*Char`s, but can fail. -//! -//! To be predictable, all errors consume one element each. -//! -//! The iterator adaptors produce neither offset nor element length to work -//! well with other adaptors, -//! while the slice iterators yield both to make more advanced use cases easy. - -use errors::{InvalidUtf8Slice, InvalidUtf16FirstUnit, Utf16PairError}; -use errors::InvalidUtf8Slice::*; -use errors::InvalidUtf8::*; -use errors::InvalidUtf8FirstByte::*; -use errors::InvalidUtf16Slice::*; -use errors::InvalidCodepoint::*; -use errors::Utf16PairError::*; -use utf8_char::Utf8Char; -use utf16_char::Utf16Char; -use traits::U16UtfExt; -extern crate core; -use self::core::borrow::Borrow; -use self::core::fmt::{self, Debug}; -use self::core::iter::Chain; -use self::core::option; - - -/// Decodes UTF-8 characters from a byte iterator into `Utf8Char`s. -/// -/// See [`IterExt::to_utf8chars()`](../trait.IterExt.html#tymethod.to_utf8chars) -/// for examples and error handling. -#[derive(Clone, Default)] -pub struct Utf8CharMerger<B:Borrow<u8>, I:Iterator<Item=B>> { - iter: I, - /// number of bytes that were read before an error was detected - after_err_leftover: u8, - /// stack because it simplifies popping. - after_err_stack: [u8; 3], -} -impl<B:Borrow<u8>, I:Iterator<Item=B>, T:IntoIterator<IntoIter=I,Item=B>> -From<T> for Utf8CharMerger<B, I> { - fn from(t: T) -> Self { - Utf8CharMerger { - iter: t.into_iter(), - after_err_leftover: 0, - after_err_stack: [0; 3], - } - } -} -impl<B:Borrow<u8>, I:Iterator<Item=B>> Utf8CharMerger<B,I> { - /// Extract the inner iterator. - /// - /// If the last item produced by `.next()` was an `Err`, - /// up to three following bytes might be missing. - /// The exact number of missing bytes for each error type should not be relied on. - /// - /// # Examples - /// - /// Three bytes swallowed: - /// ``` - /// # use encode_unicode::IterExt; - /// let mut merger = b"\xf4\xa1\xb2FS".iter().to_utf8chars(); - /// assert!(merger.next().unwrap().is_err()); - /// let mut inner: std::slice::Iter<u8> = merger.into_inner(); - /// assert_eq!(inner.next(), Some(&b'S')); // b'\xa1', b'\xb2' and b'F' disappeared - /// ``` - /// - /// All bytes present: - /// ``` - /// # use encode_unicode::IterExt; - /// let mut merger = b"\xb0FS".iter().to_utf8chars(); - /// assert!(merger.next().unwrap().is_err()); - /// assert_eq!(merger.into_inner().next(), Some(&b'F')); - /// ``` - /// - /// Two bytes missing: - /// ``` - /// # use encode_unicode::IterExt; - /// let mut merger = b"\xe0\x80\x80FS".iter().to_utf8chars(); - /// assert!(merger.next().unwrap().is_err()); - /// assert_eq!(merger.into_inner().next(), Some(&b'F')); - /// ``` - pub fn into_inner(self) -> I { - self.iter - } - - fn save(&mut self, bytes: &[u8;4], len: usize) { - // forget bytes[0] and push the others onto self.after_err_stack (in reverse). - for &after_err in bytes[1..len].iter().rev() { - self.after_err_stack[self.after_err_leftover as usize] = after_err; - self.after_err_leftover += 1; - } - } - /// Reads len-1 bytes into bytes[1..] - fn extra(&mut self, bytes: &mut[u8;4], len: usize) -> Result<(),InvalidUtf8Slice> { - // This is the only function that pushes onto after_err_stack, - // and it checks that all bytes are continuation bytes before fetching the next one. - // Therefore only the last byte retrieved can be a non-continuation byte. - // That last byte is also the last to be retrieved from after_err. - // - // Before this function is called, there has been retrieved at least one byte. - // If that byte was a continuation byte, next() produces an error - // and won't call this function. - // Therefore, we know that after_err is empty at this point. - // This means that we can use self.iter directly, and knows where to start pushing - debug_assert_eq!(self.after_err_leftover, 0, "first: {:#02x}, stack: {:?}", bytes[0], self.after_err_stack); - for i in 1..len { - if let Some(extra) = self.iter.next() { - let extra = *extra.borrow(); - bytes[i] = extra; - if extra & 0b1100_0000 != 0b1000_0000 { - // not a continuation byte - self.save(bytes, i+1); - return Err(InvalidUtf8Slice::Utf8(NotAContinuationByte(i))) - } - } else { - self.save(bytes, i); - return Err(TooShort(len)); - } - } - Ok(()) - } -} -impl<B:Borrow<u8>, I:Iterator<Item=B>> Iterator for Utf8CharMerger<B,I> { - type Item = Result<Utf8Char,InvalidUtf8Slice>; - fn next(&mut self) -> Option<Self::Item> { - let first: u8; - if self.after_err_leftover != 0 { - self.after_err_leftover -= 1; - first = self.after_err_stack[self.after_err_leftover as usize]; - } else if let Some(next) = self.iter.next() { - first = *next.borrow(); - } else { - return None; - } - - unsafe { - let mut bytes = [first, 0, 0, 0]; - let ok = match first { - 0b0000_0000...0b0111_1111 => {/*1 and */Ok(())}, - 0b1100_0010...0b1101_1111 => {//2 and not overlong - self.extra(&mut bytes, 2) // no extra validation required - }, - 0b1110_0000...0b1110_1111 => {//3 - if let Err(e) = self.extra(&mut bytes, 3) { - Err(e) - } else if bytes[0] == 0b1110_0000 && bytes[1] <= 0b10_011111 { - self.save(&bytes, 3); - Err(Utf8(OverLong)) - } else if bytes[0] == 0b1110_1101 && bytes[1] & 0b11_100000 == 0b10_100000 { - self.save(&bytes, 3); - Err(Codepoint(Utf16Reserved)) - } else { - Ok(()) - } - }, - 0b1111_0000...0b1111_0100 => {//4 - if let Err(e) = self.extra(&mut bytes, 4) { - Err(e) - } else if bytes[0] == 0b11110_000 && bytes[1] <= 0b10_001111 { - self.save(&bytes, 4); - Err(InvalidUtf8Slice::Utf8(OverLong)) - } else if bytes[0] == 0b11110_100 && bytes[1] > 0b10_001111 { - self.save(&bytes, 4); - Err(InvalidUtf8Slice::Codepoint(TooHigh)) - } else { - Ok(()) - } - }, - 0b1000_0000...0b1011_1111 => {// continuation byte - Err(Utf8(FirstByte(ContinuationByte))) - }, - 0b1100_0000...0b1100_0001 => {// 2 and overlong - Err(Utf8(OverLong)) - }, - 0b1111_0101...0b1111_0111 => {// 4 and too high codepoint - Err(Codepoint(TooHigh)) - }, - 0b1111_1000...0b1111_1111 => { - Err(Utf8(FirstByte(TooLongSeqence))) - }, - _ => unreachable!("all possible byte values should be covered") - }; - Some(ok.map(|()| Utf8Char::from_array_unchecked(bytes) )) - } - } - fn size_hint(&self) -> (usize,Option<usize>) { - let (iter_min, iter_max) = self.iter.size_hint(); - // cannot be exact, so KISS - let min = iter_min / 4; // don't bother rounding up or accounting for after_err - // handle edge case of max > usize::MAX-3 just in case. - // Using wrapping_add() wouldn't violate any API contract as the trait isn't unsafe. - let max = iter_max.and_then(|max| { - max.checked_add(self.after_err_leftover as usize) - }); - (min, max) - } -} -impl<B:Borrow<u8>, I:Iterator<Item=B>+Debug> Debug for Utf8CharMerger<B,I> { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - let mut in_order = [0u8; 3]; - for i in 0..self.after_err_leftover as usize { - in_order[i] = self.after_err_stack[self.after_err_leftover as usize - i - 1]; - } - fmtr.debug_struct("Utf8CharMerger") - .field("buffered", &&in_order[..self.after_err_leftover as usize]) - .field("inner", &self.iter) - .finish() - } -} - - -/// An [`Utf8CharMerger`](struct.Utf8CharMerger.html) that also produces -/// offsets and lengths, but can only iterate over slices. -/// -/// See [`SliceExt::utf8char_indices()`](../trait.SliceExt.html#tymethod.utf8char_indices) -/// for examples and error handling. -#[derive(Clone, Default)] -pub struct Utf8CharDecoder<'a> { - slice: &'a[u8], - index: usize, -} -impl<'a> From<&'a[u8]> for Utf8CharDecoder<'a> { - fn from(s: &[u8]) -> Utf8CharDecoder { - Utf8CharDecoder { slice: s, index: 0 } - } -} -impl<'a> Utf8CharDecoder<'a> { - /// Extract the remainder of the source slice. - /// - /// # Examples - /// - /// Unlike `Utf8CharMerger::into_inner()`, bytes directly after an error - /// are never swallowed: - /// ``` - /// # use encode_unicode::SliceExt; - /// let mut iter = b"\xf4\xa1\xb2FS".utf8char_indices(); - /// assert!(iter.next().unwrap().1.is_err()); - /// assert_eq!(iter.as_slice(), b"\xa1\xb2FS"); - /// ``` - pub fn as_slice(&self) -> &'a[u8] { - &self.slice[self.index..] - } -} -impl<'a> Iterator for Utf8CharDecoder<'a> { - type Item = (usize, Result<Utf8Char,InvalidUtf8Slice>, usize); - fn next(&mut self) -> Option<Self::Item> { - let start = self.index; - match Utf8Char::from_slice_start(&self.slice[self.index..]) { - Ok((u8c, len)) => { - self.index += len; - Some((start, Ok(u8c), len)) - }, - Err(TooShort(1)) => None, - Err(e) => { - self.index += 1; - Some((start, Err(e), 1)) - } - } - } - #[inline] - fn size_hint(&self) -> (usize,Option<usize>) { - let bytes = self.slice.len() - self.index; - // Cannot be exact, so KISS and don't bother rounding up. - // The slice is unlikely be full of 4-byte codepoints, so buffers - // allocated with the lower bound will have to be grown anyway. - (bytes/4, Some(bytes)) - } -} -impl<'a> DoubleEndedIterator for Utf8CharDecoder<'a> { - fn next_back(&mut self) -> Option<Self::Item> { - if self.index < self.slice.len() { - let extras = self.slice.iter() - .rev() - .take_while(|&b| b & 0b1100_0000 == 0b1000_0000 ) - .count(); - let starts = self.slice.len() - (extras+1); - match Utf8Char::from_slice_start(&self.slice[starts..]) { - Ok((u8c,len)) if len == 1+extras => { - self.slice = &self.slice[..starts]; - Some((starts, Ok(u8c), len)) - }, - // This enures errors for every byte in both directions, - // but means overlong and codepoint errors will be turned into - // tooshort errors. - Err(e) if extras == 0 => { - self.slice = &self.slice[..self.slice.len()-1]; - Some((self.slice.len()-1, Err(e), 1)) - }, - _ => { - self.slice = &self.slice[..self.slice.len()-1]; - Some((self.slice.len()-1, Err(Utf8(FirstByte(ContinuationByte))), 1)) - }, - } - } else { - None - } - } -} -impl<'a> Debug for Utf8CharDecoder<'a> { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - write!(fmtr, "Utf8CharDecoder {{ bytes[{}..]: {:?} }}", self.index, self.as_slice()) - } -} - - - -/// Decodes UTF-16 characters from a `u16` iterator into `Utf16Char`s. -/// -/// See [`IterExt::to_utf16chars()`](../trait.IterExt.html#tymethod.to_utf16chars) -/// for examples and error handling. -#[derive(Clone, Default)] -pub struct Utf16CharMerger<B:Borrow<u16>, I:Iterator<Item=B>> { - iter: I, - /// Used when a trailing surrogate was expected, the u16 can be any value. - prev: Option<B>, -} -impl<B:Borrow<u16>, I:Iterator<Item=B>, T:IntoIterator<IntoIter=I,Item=B>> -From<T> for Utf16CharMerger<B,I> { - fn from(t: T) -> Self { - Utf16CharMerger { iter: t.into_iter(), prev: None } - } -} -impl<B:Borrow<u16>, I:Iterator<Item=B>> Utf16CharMerger<B,I> { - /// Extract the inner iterator. - /// - /// If the last item produced was an `Err`, the first unit might be missing. - /// - /// # Examples - /// - /// Unit right after an error missing - /// ``` - /// # use encode_unicode::IterExt; - /// # use encode_unicode::error::Utf16PairError; - /// let mut merger = [0xd901, 'F' as u16, 'S' as u16].iter().to_utf16chars(); - /// assert_eq!(merger.next(), Some(Err(Utf16PairError::UnmatchedLeadingSurrogate))); - /// let mut inner: std::slice::Iter<u16> = merger.into_inner(); - /// assert_eq!(inner.next(), Some('S' as u16).as_ref()); // 'F' was consumed by Utf16CharMerger - /// ``` - /// - /// Error that doesn't swallow any units - /// ``` - /// # use encode_unicode::IterExt; - /// # use encode_unicode::error::Utf16PairError; - /// let mut merger = [0xde00, 'F' as u16, 'S' as u16].iter().to_utf16chars(); - /// assert_eq!(merger.next(), Some(Err(Utf16PairError::UnexpectedTrailingSurrogate))); - /// let mut inner: std::slice::Iter<u16> = merger.into_inner(); - /// assert_eq!(inner.next(), Some('F' as u16).as_ref()); // not consumed - /// ``` - pub fn into_inner(self) -> I { - self.iter - } - /// Returns an iterator over the remaining units. - /// Unlike `into_inner()` this will never drop any units. - /// - /// The exact type of the returned iterator should not be depended on. - /// - /// # Examples - /// - /// ``` - /// # use encode_unicode::IterExt; - /// # use encode_unicode::error::Utf16PairError; - /// let slice = [0xd901, 'F' as u16, 'S' as u16]; - /// let mut merger = slice.iter().to_utf16chars(); - /// assert_eq!(merger.next(), Some(Err(Utf16PairError::UnmatchedLeadingSurrogate))); - /// let mut remaining = merger.into_remaining_units(); - /// assert_eq!(remaining.next(), Some('F' as u16).as_ref()); - /// ``` - pub fn into_remaining_units(self) -> Chain<option::IntoIter<B>,I> { - self.prev.into_iter().chain(self.iter) - } -} -impl<B:Borrow<u16>, I:Iterator<Item=B>> Iterator for Utf16CharMerger<B,I> { - type Item = Result<Utf16Char,Utf16PairError>; - fn next(&mut self) -> Option<Self::Item> { - let first = self.prev.take().or_else(|| self.iter.next() ); - first.map(|first| unsafe { - match first.borrow().utf16_needs_extra_unit() { - Ok(false) => Ok(Utf16Char::from_array_unchecked([*first.borrow(), 0])), - Ok(true) => match self.iter.next() { - Some(second) => match second.borrow().utf16_needs_extra_unit() { - Err(InvalidUtf16FirstUnit) => Ok(Utf16Char::from_tuple_unchecked(( - *first.borrow(), - Some(*second.borrow()) - ))), - Ok(_) => { - self.prev = Some(second); - Err(Utf16PairError::UnmatchedLeadingSurrogate) - } - }, - None => Err(Utf16PairError::Incomplete) - }, - Err(InvalidUtf16FirstUnit) => Err(Utf16PairError::UnexpectedTrailingSurrogate), - } - }) - } - fn size_hint(&self) -> (usize,Option<usize>) { - let (iter_min, iter_max) = self.iter.size_hint(); - // cannot be exact, so KISS - let min = iter_min / 2; // don't bother rounding up or accounting for self.prev - let max = match (iter_max, &self.prev) { - (Some(max), &Some(_)) => max.checked_add(1), - (max, _) => max, - }; - (min, max) - } -} -impl<B:Borrow<u16>, I:Iterator<Item=B>+Debug> Debug for Utf16CharMerger<B,I> { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - fmtr.debug_struct("Utf16CharMerger") - .field("buffered", &self.prev.as_ref().map(|b| *b.borrow() )) - .field("inner", &self.iter) - .finish() - } -} - - -/// An [`Utf16CharMerger`](struct.Utf16CharMerger.html) that also produces -/// offsets and lengths, but can only iterate over slices. -/// -/// See [`SliceExt::utf16char_indices()`](../trait.SliceExt.html#tymethod.utf16char_indices) -/// for examples and error handling. -#[derive(Clone, Default)] -pub struct Utf16CharDecoder<'a> { - slice: &'a[u16], - index: usize, -} -impl<'a> From<&'a[u16]> for Utf16CharDecoder<'a> { - fn from(s: &'a[u16]) -> Self { - Utf16CharDecoder{ slice: s, index: 0 } - } -} -impl<'a> Utf16CharDecoder<'a> { - /// Extract the remainder of the source slice. - /// - /// # Examples - /// - /// Unlike `Utf16CharMerger::into_inner()`, the unit after an error is never swallowed: - /// ``` - /// # use encode_unicode::SliceExt; - /// # use encode_unicode::error::Utf16PairError; - /// let mut iter = [0xd901, 'F' as u16, 'S' as u16].utf16char_indices(); - /// assert_eq!(iter.next(), Some((0, Err(Utf16PairError::UnmatchedLeadingSurrogate), 1))); - /// assert_eq!(iter.as_slice(), &['F' as u16, 'S' as u16]); - /// ``` - pub fn as_slice(&self) -> &[u16] { - &self.slice[self.index..] - } -} -impl<'a> Iterator for Utf16CharDecoder<'a> { - type Item = (usize,Result<Utf16Char,Utf16PairError>,usize); - #[inline] - fn next(&mut self) -> Option<Self::Item> { - let start = self.index; - match Utf16Char::from_slice_start(self.as_slice()) { - Ok((u16c,len)) => { - self.index += len; - Some((start, Ok(u16c), len)) - }, - Err(EmptySlice) => None, - Err(FirstLowSurrogate) => { - self.index += 1; - Some((start, Err(UnexpectedTrailingSurrogate), 1)) - }, - Err(SecondNotLowSurrogate) => { - self.index += 1; - Some((start, Err(UnmatchedLeadingSurrogate), 1)) - }, - Err(MissingSecond) => { - self.index = self.slice.len(); - Some((start, Err(Incomplete), 1)) - } - } - } - #[inline] - fn size_hint(&self) -> (usize,Option<usize>) { - let units = self.slice.len() - self.index; - // Cannot be exact, so KISS and don't bother rounding up. - // The slice is unlikely be full of surrogate pairs, so buffers - // allocated with the lower bound will have to be grown anyway. - (units/2, Some(units)) - } -} -impl<'a> Debug for Utf16CharDecoder<'a> { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - write!(fmtr, "Utf16CharDecoder {{ units[{}..]: {:?} }}", self.index, self.as_slice()) - } -} diff --git a/vendor/encode_unicode/src/errors.rs b/vendor/encode_unicode/src/errors.rs deleted file mode 100644 index fb587fa..0000000 --- a/vendor/encode_unicode/src/errors.rs +++ /dev/null @@ -1,289 +0,0 @@ -/* Copyright 2016 The encode_unicode Developers - * - * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or - * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or - * http://opensource.org/licenses/MIT>, at your option. This file may not be - * copied, modified, or distributed except according to those terms. - */ - - -//! Boilerplatey error types - -extern crate core; -use self::core::fmt::{self,Display,Formatter}; -#[cfg(feature="std")] -use std::error::Error; - - -macro_rules! description {($err:ty, $desc:expr) => { - #[cfg(not(feature="std"))] - impl $err { - #[allow(missing_docs)] - pub fn description(&self) -> &'static str { - ($desc)(self) - } - } - #[cfg(feature="std")] - impl Error for $err { - fn description(&self) -> &'static str { - ($desc)(self) - } - } - impl Display for $err { - fn fmt(&self, fmtr: &mut Formatter) -> fmt::Result { - write!(fmtr, "{}", self.description()) - } - } -}} - - -macro_rules! single_cause {(#[$doc1:meta] #[$doc2:meta] $err:ident => $desc:expr) => { - // Rust 1.15 doesn't understand $(#[$doc:meta])* $:ident - #[$doc1] - #[$doc2] - #[derive(Clone,Copy, Debug, PartialEq,Eq)] - pub struct $err; - description!{$err, |_| $desc } -}} - - -single_cause!{ - /// Cannot tell whether an `u16` needs an extra unit, - /// because it's a trailing surrogate itself. - InvalidUtf16FirstUnit => "is a trailing surrogate" -} - -single_cause!{ - /// Cannot create an `Utf8Char` or `Utf16Char` from the first codepoint of a str, - /// because there are none. - EmptyStrError => "is empty" -} - -single_cause!{ - /// Cannot create an `Utf8Char` from a standalone `u8` - /// that is not an ASCII character. - NonAsciiError => "is not an ASCII character" -} - -single_cause!{ - /// Cannot create an `Utf16Char` from a standalone `u16` that is not a - /// codepoint in the basic multilingual plane, but part of a suurrogate pair. - NonBMPError => "is not a codepoint in the basic multilingual plane" -} - - - -macro_rules! simple {(#[$tydoc:meta] $err:ident { - $($(#[$vardoc:meta])* ::$variant:ident => $string:expr),+, - } ) => { - #[$tydoc] - #[derive(Clone,Copy, Debug, PartialEq,Eq)] - pub enum $err { - $($(#[$vardoc])* $variant),* - } - description!{$err, |e: &$err| match *e {$($err::$variant=>$string),*} } -}} - - -simple!{/// Reasons why an `u32` is not a valid UTF codepoint. - InvalidCodepoint { - /// It's reserved for UTF-16 surrogate pairs." - ::Utf16Reserved => "is reserved for UTF-16 surrogate pairs", - /// It's higher than the highest codepoint (which is 0x10ffff). - ::TooHigh => "is higher than the highest codepoint", - }} -use self::InvalidCodepoint::*; -impl InvalidCodepoint { - /// Get the range of values for which this error would be given. - pub fn error_range(self) -> (u32,u32) {match self { - Utf16Reserved => (0xd8_00, 0xdf_ff), - TooHigh => (0x00_10_ff_ff, 0xff_ff_ff_ff), - }} -} - - -simple!{/// Reasons why a `[u16; 2]` doesn't form a valid UTF-16 codepoint. - InvalidUtf16Array { - /// The first unit is a trailing/low surrogate, which is never valid. - ::FirstIsTrailingSurrogate => "the first unit is a trailing surrogate, which is never valid", - /// The second unit is needed, but is not a trailing surrogate. - ::SecondIsNotTrailingSurrogate => "the second unit is needed but is not a trailing surrogate", - }} - -simple!{/// Reasons why one or two `u16`s are not valid UTF-16, in sinking precedence. - InvalidUtf16Tuple { - /// The first unit is a trailing/low surrogate, which is never valid. - /// - /// Note that the value of a low surrogate is actually higher than a high surrogate. - ::FirstIsTrailingSurrogate => "the first unit is a trailing / low surrogate, which is never valid", - /// You provided a second unit, but the first one stands on its own. - ::SuperfluousSecond => "the second unit is superfluous", - /// The first and only unit requires a second unit. - ::MissingSecond => "the first unit requires a second unit", - /// The first unit requires a second unit, but it's not a trailing/low surrogate. - /// - /// Note that the value of a low surrogate is actually higher than a high surrogate. - ::InvalidSecond => "the required second unit is not a trailing / low surrogate", - }} - - -simple!{/// Reasons why a slice of `u16`s doesn't start with valid UTF-16. - InvalidUtf16Slice { - /// The slice is empty. - ::EmptySlice => "the slice is empty", - /// The first unit is a low surrogate. - ::FirstLowSurrogate => "the first unit is a trailing surrogate", - /// The first and only unit requires a second unit. - ::MissingSecond => "the first and only unit requires a second one", - /// The first unit requires a second one, but it's not a trailing surrogate. - ::SecondNotLowSurrogate => "the required second unit is not a trailing surrogate", - }} - -simple!{/// Types of invalid sequences encountered by `Utf16CharParser`. - Utf16PairError { - /// A trailing surrogate was not preceeded by a leading surrogate. - ::UnexpectedTrailingSurrogate => "a trailing surrogate was not preceeded by a leading surrogate", - /// A leading surrogate was followed by an unit that was not a trailing surrogate. - ::UnmatchedLeadingSurrogate => "a leading surrogate was followed by an unit that was not a trailing surrogate", - /// A trailing surrogate was expected when the end was reached. - ::Incomplete => "a trailing surrogate was expected when the end was reached", - }} - - -simple!{/// Reasons why `Utf8Char::from_str()` or `Utf16Char::from_str()` failed. - FromStrError { - /// `Utf8Char` or `Utf16Char` cannot store more than a single codepoint. - ::MultipleCodepoints => "has more than one codepoint", - /// `Utf8Char` or `Utf16Char` cannot be empty. - ::Empty => "is empty", - }} - - -simple!{/// Reasons why a byte is not the start of a UTF-8 codepoint. - InvalidUtf8FirstByte { - /// Sequences cannot be longer than 4 bytes. Is given for values >= 240. - ::TooLongSeqence => "is greater than 247 (UTF-8 sequences cannot be longer than four bytes)", - /// This byte belongs to a previous sequence. Is given for values between 128 and 192 (exclusive). - ::ContinuationByte => "is a continuation of a previous sequence", - }} -use self::InvalidUtf8FirstByte::*; - - - -macro_rules! complex { -($err:ty - {$($sub:ty => $to:expr,)*} - {$($desc:pat => $string:expr),+,} - => $use_cause:expr => - {$($cause:pat => $result:expr),+,} $(#[$causedoc:meta])* -) => { - $(impl From<$sub> for $err { - fn from(error: $sub) -> $err { - $to(error) - } - })* - #[cfg(not(feature="std"))] - impl $err { - #[allow(missing_docs)] - pub fn description(&self) -> &'static str { - match *self{ $($desc => $string,)* } - } - /// A hack to avoid two Display impls - fn cause(&self) -> Option<&Display> {None} - } - #[cfg(feature="std")] - impl Error for $err { - fn description(&self) -> &'static str { - match *self{ $($desc => $string,)* } - } - $(#[$causedoc])* - fn cause(&self) -> Option<&Error> { - match *self{ $($cause => $result,)* } - } - } - impl Display for $err { - fn fmt(&self, fmtr: &mut Formatter) -> fmt::Result { - match (self.cause(), $use_cause) { - (Some(d),true) => write!(fmtr, "{}: {}", self.description(), d), - _ => write!(fmtr, "{}", self.description()), - } - } - } -}} - - -/// Reasons why a byte sequence is not valid UTF-8, excluding invalid codepoint. -/// In sinking precedence. -#[derive(Clone,Copy, Debug, PartialEq,Eq)] -pub enum InvalidUtf8 { - /// Something is wrong with the first byte. - FirstByte(InvalidUtf8FirstByte), - /// The byte at index 1...3 should be a continuation byte, - /// but dosesn't fit the pattern 0b10xx_xxxx. - NotAContinuationByte(usize), - /// There are too many leading zeros: it could be a byte shorter. - /// - /// [Decoding this could allow someone to input otherwise prohibited - /// characters and sequences, such as "../"](https://tools.ietf.org/html/rfc3629#section-10). - OverLong, -} -use self::InvalidUtf8::*; -complex!{InvalidUtf8 { - InvalidUtf8FirstByte => FirstByte, - } { - FirstByte(TooLongSeqence) => "the first byte is greater than 239 (UTF-8 sequences cannot be longer than four bytes)", - FirstByte(ContinuationByte) => "the first byte is a continuation of a previous sequence", - OverLong => "the sequence contains too many zeros and could be shorter", - NotAContinuationByte(_) => "the sequence is too short", - } => false => { - FirstByte(ref cause) => Some(cause), - _ => None, - }/// Returns `Some` if the error is a `InvalidUtf8FirstByte`. -} - - -/// Reasons why a byte array is not valid UTF-8, in sinking precedence. -#[derive(Clone,Copy, Debug, PartialEq,Eq)] -pub enum InvalidUtf8Array { - /// Not a valid UTF-8 sequence. - Utf8(InvalidUtf8), - /// Not a valid unicode codepoint. - Codepoint(InvalidCodepoint), -} -complex!{InvalidUtf8Array { - InvalidUtf8 => InvalidUtf8Array::Utf8, - InvalidCodepoint => InvalidUtf8Array::Codepoint, - } { - InvalidUtf8Array::Utf8(_) => "the sequence is invalid UTF-8", - InvalidUtf8Array::Codepoint(_) => "the encoded codepoint is invalid", - } => true => { - InvalidUtf8Array::Utf8(ref u) => Some(u), - InvalidUtf8Array::Codepoint(ref c) => Some(c), - }/// Always returns `Some`. -} - - -/// Reasons why a byte slice is not valid UTF-8, in sinking precedence. -#[derive(Clone,Copy, Debug, PartialEq,Eq)] -pub enum InvalidUtf8Slice { - /// Something is certainly wrong with the first byte. - Utf8(InvalidUtf8), - /// The encoded codepoint is invalid: - Codepoint(InvalidCodepoint), - /// The slice is too short; n bytes was required. - TooShort(usize), -} -complex!{InvalidUtf8Slice { - InvalidUtf8 => InvalidUtf8Slice::Utf8, - InvalidCodepoint => InvalidUtf8Slice::Codepoint, - } { - InvalidUtf8Slice::Utf8(_) => "the sequence is invalid UTF-8", - InvalidUtf8Slice::Codepoint(_) => "the encoded codepoint is invalid", - InvalidUtf8Slice::TooShort(1) => "the slice is empty", - InvalidUtf8Slice::TooShort(_) => "the slice is shorter than the sequence", - } => true => { - InvalidUtf8Slice::Utf8(ref u) => Some(u), - InvalidUtf8Slice::Codepoint(ref c) => Some(c), - InvalidUtf8Slice::TooShort(_) => None, - } -} diff --git a/vendor/encode_unicode/src/lib.rs b/vendor/encode_unicode/src/lib.rs deleted file mode 100644 index c50e126..0000000 --- a/vendor/encode_unicode/src/lib.rs +++ /dev/null @@ -1,78 +0,0 @@ -/* Copyright 2016 The encode_unicode Developers - * - * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or - * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or - * http://opensource.org/licenses/MIT>, at your option. This file may not be - * copied, modified, or distributed except according to those terms. - */ - - -/*! -Miscellaneous UTF-8 and UTF-16 types and methods. - -# Optional features: -* `#![no_std]`-mode: There are a few differences: - * `Error` doesn't exist, but `description()` is made available as an inherent impl. - * `Extend`/`FromIterator`-implementations for `String`/`Vec<u8>`/`Vec<u16>` are missing. - * There is no `io`, so `Utf8Iterator` and `Utf8CharSplitter` doesn't implement `Read`. - - This feature is enabled by setting `default-features=false` in `Cargo.toml`: - `encode_unicode = {version="0.3.4", default-features=false}` -* Integration with the [ascii](https://tomprogrammer.github.io/rust-ascii/ascii/index.html) crate: - Convert `Utf8Char` and `Utf16Char` to and from - [ascii::`AsciiChar`](https://tomprogrammer.github.io/rust-ascii/ascii/enum.AsciiChar.html). - -The minimum supported version of Rust is 1.15, -older versions might work now but can break with a minor update. - -[crates.io page](https://crates.io/crates/encode_unicode) -[github repository](https://github.com/tormol/encode_unicode) - -*/ - -#![warn(missing_docs)] - -#![cfg_attr(not(feature="std"), no_std)] -// either `cargo clippy` doesn't see theese, or I get a warning when I build. -#![cfg_attr(feature="clippy", feature(plugin))] -#![cfg_attr(feature="clippy", plugin(clippy))] -#![cfg_attr(feature="clippy", allow(derive_hash_xor_eq))]// tested -#![cfg_attr(feature="clippy", allow(len_without_is_empty))]// UtfxChar is never empty -#![cfg_attr(feature="clippy", allow(match_same_arms))]// looks better IMO -#![cfg_attr(feature="clippy", allow(needless_return))]// `foo.bar(); foo` looks unfinished -#![cfg_attr(feature="clippy", allow(redundant_closure))]// keep it explicit -#![cfg_attr(feature="clippy", allow(redundant_closure_call))]// not redundant in macros -#![cfg_attr(feature="clippy", allow(cast_lossless))]// too much noise (and too verbose) -// precedence: I prefer spaces to parentheses, but it's nice to recheck. - -mod errors; -mod traits; -mod utf8_char; -mod utf8_iterators; -mod utf16_char; -mod utf16_iterators; -mod decoding_iterators; - -pub use traits::{CharExt, U8UtfExt, U16UtfExt, StrExt, IterExt, SliceExt}; -pub use utf8_char::Utf8Char; -pub use utf16_char::Utf16Char; -pub use utf8_iterators::{Utf8Iterator, iter_bytes}; -pub use utf16_iterators::{Utf16Iterator, iter_units}; - -pub mod error {// keeping the public interface in one file - //! Errors returned by various conversion methods in this crate. - pub use errors::{FromStrError, EmptyStrError}; - pub use errors::{InvalidCodepoint, InvalidUtf8}; - pub use errors::{InvalidUtf8FirstByte,InvalidUtf16FirstUnit}; - pub use errors::{InvalidUtf8Slice,InvalidUtf16Slice}; - pub use errors::{InvalidUtf8Array,InvalidUtf16Array,InvalidUtf16Tuple}; - pub use errors::Utf16PairError; -} - -pub mod iterator { - //! Iterator types that you should rarely need to name - pub use utf8_iterators::{Utf8Iterator, Utf8CharSplitter, Utf8Chars, Utf8CharIndices}; - pub use utf16_iterators::{Utf16Iterator, Utf16CharSplitter, Utf16Chars, Utf16CharIndices}; - pub use decoding_iterators::{Utf8CharMerger, Utf8CharDecoder}; - pub use decoding_iterators::{Utf16CharMerger, Utf16CharDecoder}; -} diff --git a/vendor/encode_unicode/src/traits.rs b/vendor/encode_unicode/src/traits.rs deleted file mode 100644 index 3f36903..0000000 --- a/vendor/encode_unicode/src/traits.rs +++ /dev/null @@ -1,1014 +0,0 @@ -/* Copyright 2016 The encode_unicode Developers - * - * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or - * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or - * http://opensource.org/licenses/MIT>, at your option. This file may not be - * copied, modified, or distributed except according to those terms. - */ - -#![allow(unused_unsafe)]// explicit unsafe{} blocks in unsafe functions are a good thing. - -use utf8_char::Utf8Char; -use utf16_char::Utf16Char; -use utf8_iterators::*; -use utf16_iterators::*; -use decoding_iterators::*; -use error::*; -extern crate core; -use self::core::{char, u32, mem}; -use self::core::ops::{Not, Index, RangeFull}; -use self::core::borrow::Borrow; -#[cfg(feature="ascii")] -extern crate ascii; -#[cfg(feature="ascii")] -use self::ascii::AsciiStr; - -// TODO better docs and tests - -/// Methods for working with `u8`s as UTF-8 bytes. -pub trait U8UtfExt { - /// How many more bytes will you need to complete this codepoint? - /// - /// # Errors - /// - /// An error is returned if the byte is not a valid start of an UTF-8 - /// codepoint: - /// - /// * `128..192`: ContinuationByte - /// * `248..`: TooLongSequence - /// - /// Values in 244..248 represent a too high codepoint, but do not cause an - /// error. - fn extra_utf8_bytes(self) -> Result<usize,InvalidUtf8FirstByte>; - - /// How many more bytes will you need to complete this codepoint? - /// - /// This function assumes that the byte is a valid UTF-8 start, and might - /// return any value otherwise. (but the function is pure and safe to call - /// with any value). - fn extra_utf8_bytes_unchecked(self) -> usize; -} - -impl U8UtfExt for u8 { - #[inline] - fn extra_utf8_bytes(self) -> Result<usize,InvalidUtf8FirstByte> { - use error::InvalidUtf8FirstByte::{ContinuationByte,TooLongSeqence}; - // the bit twiddling is explained in extra_utf8_bytes_unchecked() - if self < 128 { - return Ok(0); - } - match ((self as u32)<<25).not().leading_zeros() { - n @ 1...3 => Ok(n as usize), - 0 => Err(ContinuationByte), - _ => Err(TooLongSeqence), - } - } - #[inline] - fn extra_utf8_bytes_unchecked(self) -> usize { - // For fun I've optimized this function (for x86 instruction count): - // The most straightforward implementation, that lets the compiler do - // the optimizing: - //match self { - // 0b0000_0000...0b0111_1111 => 0, - // 0b1100_0010...0b1101_1111 => 1, - // 0b1110_0000...0b1110_1111 => 2, - // 0b1111_0000...0b1111_0100 => 3, - // _ => whatever() - //} - // Using `unsafe{self::core::hint::unreachable_unchecked()}` for the - // "don't care" case is a terrible idea: while having the function - // non-deterministically return whatever happens to be in a register - // MIGHT be acceptable, it permits the function to not `ret`urn at all, - // but let execution fall through to whatever comes after it in the - // binary! (in other words completely UB). - // Currently unreachable_unchecked() might trap too, - // which is certainly not what we want. - // I also think `unsafe{mem::unitialized()}` is much more likely to - // explicitly produce whatever happens to be in a register than tell - // the compiler it can ignore this branch but needs to produce a value. - // - // From the bit patterns we see that for non-ASCII values the result is - // (number of leading set bits) - 1 - // The standard library doesn't have a method for counting leading ones, - // but it has leading_zeros(), which can be used after inverting. - // This function can therefore be reduced to the one-liner - //`self.not().leading_zeros().saturating_sub(1) as usize`, which would - // be branchless for architectures with instructions for - // leading_zeros() and saturating_sub(). - - // Shortest version as long as ASCII-ness can be predicted: (especially - // if the BSR instruction which leading_zeros() uses is microcoded or - // doesn't exist) - // u8.leading_zeros() would cast to a bigger type internally, so that's - // free. compensating by shifting left by 24 before inverting lets the - // compiler know that the value passed to leading_zeros() is not zero, - // for which BSR's output is undefined and leading_zeros() normally has - // special case with a branch. - // Shifting one bit too many left acts as a saturating_sub(1). - if self<128 {0} else {((self as u32)<<25).not().leading_zeros() as usize} - - // Branchless but longer version: (9 instructions) - // It's tempting to try (self|0x80).not().leading_zeros().wrapping_sub(1), - // but that produces high lengths for ASCII values 0b01xx_xxxx. - // If we could somehow (branchlessy) clear that bit for ASCII values... - // We can by masking with the value shifted right with sign extension! - // (any nonzero number of bits in range works) - //let extended = self as i8 as i32; - //let ascii_cleared = (extended<<25) & (extended>>25); - //ascii_cleared.not().leading_zeros() as usize - - // cmov version: (7 instructions) - //(((self as u32)<<24).not().leading_zeros() as usize).saturating_sub(1) - } -} - - -/// Methods for working with `u16`s as UTF-16 units. -pub trait U16UtfExt { - /// Will you need an extra unit to complete this codepoint? - /// - /// Returns `Err` for trailing surrogates, `Ok(true)` for leading surrogates, - /// and `Ok(false)` for others. - fn utf16_needs_extra_unit(self) -> Result<bool,InvalidUtf16FirstUnit>; - - /// Does this `u16` need another `u16` to complete a codepoint? - /// Returns `(self & 0xfc00) == 0xd800` - /// - /// Is basically an unchecked variant of `utf16_needs_extra_unit()`. - fn is_utf16_leading_surrogate(self) -> bool; -} -impl U16UtfExt for u16 { - #[inline] - fn utf16_needs_extra_unit(self) -> Result<bool,InvalidUtf16FirstUnit> { - match self { - // https://en.wikipedia.org/wiki/UTF-16#U.2B10000_to_U.2B10FFFF - 0x00_00...0xd7_ff | 0xe0_00...0xff_ff => Ok(false), - 0xd8_00...0xdb_ff => Ok(true), - _ => Err(InvalidUtf16FirstUnit) - } - } - #[inline] - fn is_utf16_leading_surrogate(self) -> bool { - (self & 0xfc00) == 0xd800// Clear the ten content bytes of a surrogate, - // and see if it's a leading surrogate. - } -} - - - - -/// Extension trait for `char` that adds methods for converting to and from UTF-8 or UTF-16. -pub trait CharExt: Sized { - /// Get the UTF-8 representation of this codepoint. - /// - /// `Utf8Char` is to `[u8;4]` what `char` is to `u32`: - /// a restricted type that cannot be mutated internally. - fn to_utf8(self) -> Utf8Char; - - /// Get the UTF-16 representation of this codepoint. - /// - /// `Utf16Char` is to `[u16;2]` what `char` is to `u32`: - /// a restricted type that cannot be mutated internally. - fn to_utf16(self) -> Utf16Char; - - /// Iterate over or [read](https://doc.rust-lang.org/std/io/trait.Read.html) - /// the one to four bytes in the UTF-8 representation of this codepoint. - /// - /// An identical alternative to the unstable `char.encode_utf8()`. - /// That method somehow still exist on stable, so I have to use a different name. - fn iter_utf8_bytes(self) -> Utf8Iterator; - - /// Iterate over the one or two units in the UTF-16 representation of this codepoint. - /// - /// An identical alternative to the unstable `char.encode_utf16()`. - /// That method somehow still exist on stable, so I have to use a different name. - fn iter_utf16_units(self) -> Utf16Iterator; - - - /// Convert this char to an UTF-8 array, and also return how many bytes of - /// the array are used, - /// - /// The returned array is left-aligned with unused bytes set to zero. - fn to_utf8_array(self) -> ([u8; 4], usize); - - /// Convert this `char` to UTF-16. - /// - /// The second element is non-zero when a surrogate pair is required. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::CharExt; - /// - /// assert_eq!('@'.to_utf16_array(), ['@' as u16, 0]); - /// assert_eq!('睷'.to_utf16_array(), ['睷' as u16, 0]); - /// assert_eq!('\u{abcde}'.to_utf16_array(), [0xda6f, 0xdcde]); - /// ``` - fn to_utf16_array(self) -> [u16; 2]; - - /// Convert this `char` to UTF-16. - /// The second item is `Some` if a surrogate pair is required. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::CharExt; - /// - /// assert_eq!('@'.to_utf16_tuple(), ('@' as u16, None)); - /// assert_eq!('睷'.to_utf16_tuple(), ('睷' as u16, None)); - /// assert_eq!('\u{abcde}'.to_utf16_tuple(), (0xda6f, Some(0xdcde))); - /// ``` - fn to_utf16_tuple(self) -> (u16, Option<u16>); - - - - /// Create a `char` from the start of an UTF-8 slice, - /// and also return how many bytes were used. - /// - /// # Errors - /// - /// Returns an `Err` if the slice is empty, doesn't start with a valid - /// UTF-8 sequence or is too short for the sequence. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::CharExt; - /// use encode_unicode::error::InvalidUtf8Slice::*; - /// use encode_unicode::error::InvalidUtf8::*; - /// - /// assert_eq!(char::from_utf8_slice_start(&[b'A', b'B', b'C']), Ok(('A',1))); - /// assert_eq!(char::from_utf8_slice_start(&[0xdd, 0xbb]), Ok(('\u{77b}',2))); - /// - /// assert_eq!(char::from_utf8_slice_start(&[]), Err(TooShort(1))); - /// assert_eq!(char::from_utf8_slice_start(&[0xf0, 0x99]), Err(TooShort(4))); - /// assert_eq!(char::from_utf8_slice_start(&[0xee, b'F', 0x80]), Err(Utf8(NotAContinuationByte(1)))); - /// assert_eq!(char::from_utf8_slice_start(&[0xee, 0x99, 0x0f]), Err(Utf8(NotAContinuationByte(2)))); - /// ``` - fn from_utf8_slice_start(src: &[u8]) -> Result<(Self,usize),InvalidUtf8Slice>; - - /// Create a `char` from the start of an UTF-16 slice, - /// and also return how many units were used. - /// - /// If you want to continue after an error, continue with the next `u16` unit. - fn from_utf16_slice_start(src: &[u16]) -> Result<(Self,usize), InvalidUtf16Slice>; - - - /// Convert an UTF-8 sequence as returned from `.to_utf8_array()` into a `char` - /// - /// The codepoint must start at the first byte, and leftover bytes are ignored. - /// - /// # Errors - /// - /// Returns an `Err` if the array doesn't start with a valid UTF-8 sequence. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::CharExt; - /// use encode_unicode::error::InvalidUtf8Array::*; - /// use encode_unicode::error::InvalidUtf8::*; - /// use encode_unicode::error::InvalidCodepoint::*; - /// - /// assert_eq!(char::from_utf8_array([b'A', 0, 0, 0]), Ok('A')); - /// assert_eq!(char::from_utf8_array([0xf4, 0x8b, 0xbb, 0xbb]), Ok('\u{10befb}')); - /// assert_eq!(char::from_utf8_array([b'A', b'B', b'C', b'D']), Ok('A')); - /// assert_eq!(char::from_utf8_array([0, 0, 0xcc, 0xbb]), Ok('\0')); - /// - /// assert_eq!(char::from_utf8_array([0xef, b'F', 0x80, 0x80]), Err(Utf8(NotAContinuationByte(1)))); - /// assert_eq!(char::from_utf8_array([0xc1, 0x80, 0, 0]), Err(Utf8(OverLong))); - /// assert_eq!(char::from_utf8_array([0xf7, 0xaa, 0x99, 0x88]), Err(Codepoint(TooHigh))); - /// ``` - fn from_utf8_array(utf8: [u8; 4]) -> Result<Self,InvalidUtf8Array>; - - /// Convert a UTF-16 pair as returned from `.to_utf16_array()` into a `char`. - /// - /// The second element is ignored when not required. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::CharExt; - /// use encode_unicode::error::InvalidUtf16Array; - /// - /// assert_eq!(char::from_utf16_array(['x' as u16, 'y' as u16]), Ok('x')); - /// assert_eq!(char::from_utf16_array(['睷' as u16, 0]), Ok('睷')); - /// assert_eq!(char::from_utf16_array([0xda6f, 0xdcde]), Ok('\u{abcde}')); - /// assert_eq!(char::from_utf16_array([0xf111, 0xdbad]), Ok('\u{f111}')); - /// assert_eq!(char::from_utf16_array([0xdaaf, 0xdaaf]), Err(InvalidUtf16Array::SecondIsNotTrailingSurrogate)); - /// assert_eq!(char::from_utf16_array([0xdcac, 0x9000]), Err(InvalidUtf16Array::FirstIsTrailingSurrogate)); - /// ``` - fn from_utf16_array(utf16: [u16; 2]) -> Result<Self, InvalidUtf16Array>; - - /// Convert a UTF-16 pair as returned from `.to_utf16_tuple()` into a `char`. - fn from_utf16_tuple(utf16: (u16, Option<u16>)) -> Result<Self, InvalidUtf16Tuple>; - - - /// Convert an UTF-8 sequence into a char. - /// - /// The length of the slice is taken as length of the sequence; - /// it should be 1,2,3 or 4. - /// - /// # Safety - /// - /// The slice must contain exactly one, valid, UTF-8 sequence. - /// - /// Passing a slice that produces an invalid codepoint is always undefined - /// behavior; Later checks that the codepoint is valid can be removed - /// by the compiler. - /// - /// # Panics - /// - /// If the slice is empty - unsafe fn from_utf8_exact_slice_unchecked(src: &[u8]) -> Self; - - /// Convert a UTF-16 array as returned from `.to_utf16_array()` into a - /// `char`. - /// - /// This function is safe because it avoids creating invalid codepoints, - /// but the returned value might not be what one expectedd. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::CharExt; - /// - /// // starts with a trailing surrogate - converted as if it was a valid - /// // surrogate pair anyway. - /// assert_eq!(char::from_utf16_array_unchecked([0xdbad, 0xf19e]), '\u{fb59e}'); - /// // missing trailing surrogate - ditto - /// assert_eq!(char::from_utf16_array_unchecked([0xd802, 0]), '\u{10800}'); - /// ``` - fn from_utf16_array_unchecked(utf16: [u16;2]) -> Self; - - /// Convert a UTF-16 tuple as returned from `.to_utf16_tuple()` into a `char`. - unsafe fn from_utf16_tuple_unchecked(utf16: (u16, Option<u16>)) -> Self; - - - /// Produces more detailed errors than `char::from_u32()` - /// - /// # Errors - /// - /// This function will return an error if - /// - /// * the value is greater than 0x10ffff - /// * the value is between 0xd800 and 0xdfff (inclusive) - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::CharExt; - /// use encode_unicode::error::InvalidCodepoint; - /// - /// assert_eq!(char::from_u32_detailed(0x41), Ok('A')); - /// assert_eq!(char::from_u32_detailed(0x40_00_00), Err(InvalidCodepoint::TooHigh)); - /// assert_eq!(char::from_u32_detailed(0xd951), Err(InvalidCodepoint::Utf16Reserved)); - /// assert_eq!(char::from_u32_detailed(0xdddd), Err(InvalidCodepoint::Utf16Reserved)); - /// assert_eq!(char::from_u32_detailed(0xdd), Ok('Ý')); - /// assert_eq!(char::from_u32_detailed(0x1f331), Ok('🌱')); - /// ``` - fn from_u32_detailed(c: u32) -> Result<Self,InvalidCodepoint>; -} - - - -impl CharExt for char { - ///////// - //UTF-8// - ///////// - - fn to_utf8(self) -> Utf8Char { - self.into() - } - fn iter_utf8_bytes(self) -> Utf8Iterator { - self.to_utf8().into_iter() - } - - fn to_utf8_array(self) -> ([u8; 4], usize) { - let len = self.len_utf8(); - let mut c = self as u32; - if len == 1 {// ASCII, the common case - ([c as u8, 0, 0, 0], 1) - } else { - let mut parts = 0;// convert to 6-bit bytes - parts |= c & 0x3f; c>>=6; - parts<<=8; parts |= c & 0x3f; c>>=6; - parts<<=8; parts |= c & 0x3f; c>>=6; - parts<<=8; parts |= c & 0x3f; - parts |= 0x80_80_80_80;// set the most significant bit - parts >>= 8*(4-len);// right-align bytes - // Now, unused bytes are zero, (which matters for Utf8Char.eq()) - // and the rest are 0b10xx_xxxx - - // set header on first byte - parts |= (0xff_00u32 >> len) & 0xff;// store length - parts &= Not::not(1u32 << 7-len);// clear the next bit after it - - let bytes: [u8; 4] = unsafe{ mem::transmute(u32::from_le(parts)) }; - (bytes, len) - } - } - - - fn from_utf8_slice_start(src: &[u8]) -> Result<(Self,usize),InvalidUtf8Slice> { - use errors::InvalidUtf8::*; - use errors::InvalidUtf8Slice::*; - let first = match src.first() { - Some(first) => *first, - None => return Err(TooShort(1)), - }; - let bytes = match first.extra_utf8_bytes() { - Err(e) => return Err(Utf8(FirstByte(e))), - Ok(0) => return Ok((first as char, 1)), - Ok(extra) if extra >= src.len() - => return Err(TooShort(extra+1)), - Ok(extra) => &src[..extra+1], - }; - if let Some(i) = bytes.iter().skip(1).position(|&b| (b >> 6) != 0b10 ) { - Err(Utf8(NotAContinuationByte(i+1))) - } else if overlong(bytes[0], bytes[1]) { - Err(Utf8(OverLong)) - } else { - match char::from_u32_detailed(merge_nonascii_unchecked_utf8(bytes)) { - Ok(c) => Ok((c, bytes.len())), - Err(e) => Err(Codepoint(e)), - } - } - } - - fn from_utf8_array(utf8: [u8; 4]) -> Result<Self,InvalidUtf8Array> { - use errors::InvalidUtf8::*; - use errors::InvalidUtf8Array::*; - let src = match utf8[0].extra_utf8_bytes() { - Err(error) => return Err(Utf8(FirstByte(error))), - Ok(0) => return Ok(utf8[0] as char), - Ok(extra) => &utf8[..extra+1], - }; - if let Some(i) = src[1..].iter().position(|&b| (b >> 6) != 0b10 ) { - Err(Utf8(NotAContinuationByte(i+1))) - } else if overlong(utf8[0], utf8[1]) { - Err(Utf8(OverLong)) - } else { - char::from_u32_detailed(merge_nonascii_unchecked_utf8(src)) - .map_err(|e| Codepoint(e) ) - } - } - - unsafe fn from_utf8_exact_slice_unchecked(src: &[u8]) -> Self { - if src.len() == 1 { - src[0] as char - } else { - char::from_u32_unchecked(merge_nonascii_unchecked_utf8(src)) - } - } - - - - ////////// - //UTF-16// - ////////// - - fn to_utf16(self) -> Utf16Char { - Utf16Char::from(self) - } - fn iter_utf16_units(self) -> Utf16Iterator { - self.to_utf16().into_iter() - } - - fn to_utf16_array(self) -> [u16;2] { - let (first, second) = self.to_utf16_tuple(); - [first, second.unwrap_or(0)] - } - fn to_utf16_tuple(self) -> (u16, Option<u16>) { - if self <= '\u{ffff}' {// single - (self as u16, None) - } else {// double - let c = self as u32 - 0x_01_00_00; - let high = 0x_d8_00 + (c >> 10); - let low = 0x_dc_00 + (c & 0x_03_ff); - (high as u16, Some(low as u16)) - } - } - - - fn from_utf16_slice_start(src: &[u16]) -> Result<(Self,usize), InvalidUtf16Slice> { - use errors::InvalidUtf16Slice::*; - unsafe {match (src.get(0), src.get(1)) { - (Some(&u @ 0x00_00...0xd7_ff), _) | - (Some(&u @ 0xe0_00...0xff_ff), _) - => Ok((char::from_u32_unchecked(u as u32), 1)), - (Some(&0xdc_00...0xdf_ff), _) => Err(FirstLowSurrogate), - (None, _) => Err(EmptySlice), - (Some(&f @ 0xd8_00...0xdb_ff), Some(&s @ 0xdc_00...0xdf_ff)) - => Ok((char::from_utf16_tuple_unchecked((f, Some(s))), 2)), - (Some(&0xd8_00...0xdb_ff), Some(_)) => Err(SecondNotLowSurrogate), - (Some(&0xd8_00...0xdb_ff), None) => Err(MissingSecond), - (Some(_), _) => unreachable!() - }} - } - - fn from_utf16_array(utf16: [u16;2]) -> Result<Self, InvalidUtf16Array> { - use errors::InvalidUtf16Array::*; - if let Some(c) = char::from_u32(utf16[0] as u32) { - Ok(c) // single - } else if utf16[0] < 0xdc_00 && utf16[1] & 0xfc_00 == 0xdc_00 { - // correct surrogate pair - Ok(combine_surrogates(utf16[0], utf16[1])) - } else if utf16[0] < 0xdc_00 { - Err(SecondIsNotTrailingSurrogate) - } else { - Err(FirstIsTrailingSurrogate) - } - } - fn from_utf16_tuple(utf16: (u16, Option<u16>)) -> Result<Self, InvalidUtf16Tuple> { - use errors::InvalidUtf16Tuple::*; - unsafe{ match utf16 { - (0x00_00...0xd7_ff, None) | // single - (0xe0_00...0xff_ff, None) | // single - (0xd8_00...0xdb_ff, Some(0xdc_00...0xdf_ff)) // correct surrogate - => Ok(char::from_utf16_tuple_unchecked(utf16)), - (0xd8_00...0xdb_ff, Some(_)) => Err(InvalidSecond), - (0xd8_00...0xdb_ff, None ) => Err(MissingSecond), - (0xdc_00...0xdf_ff, _ ) => Err(FirstIsTrailingSurrogate), - ( _ , Some(_)) => Err(SuperfluousSecond), - ( _ , None ) => unreachable!() - }} - } - - fn from_utf16_array_unchecked(utf16: [u16;2]) -> Self { - // treat any array with a surrogate value in [0] as a surrogate because - // combine_surrogates() is safe. - // `(utf16[0] & 0xf800) == 0xd80` might not be quite as fast as - // `utf16[1] != 0`, but avoiding the potential for UB is worth it - // since the conversion isn't zero-cost in either case. - char::from_u32(utf16[0] as u32) - .unwrap_or_else(|| combine_surrogates(utf16[0], utf16[1]) ) - } - unsafe fn from_utf16_tuple_unchecked(utf16: (u16, Option<u16>)) -> Self { - match utf16.1 { - Some(second) => combine_surrogates(utf16.0, second), - None => char::from_u32_unchecked(utf16.0 as u32) - } - } - - - fn from_u32_detailed(c: u32) -> Result<Self,InvalidCodepoint> { - match char::from_u32(c) { - Some(c) => Ok(c), - None if c > 0x10_ff_ff => Err(InvalidCodepoint::TooHigh), - None => Err(InvalidCodepoint::Utf16Reserved), - } - } -} - -// Adapted from https://www.cl.cam.ac.uk/~mgk25/ucs/utf8_check.c -fn overlong(first: u8, second: u8) -> bool { - if first < 0x80 { - false - } else if (first & 0xe0) == 0xc0 { - (first & 0xfe) == 0xc0 - } else if (first & 0xf0) == 0xe0 { - first == 0xe0 && (second & 0xe0) == 0x80 - } else { - first == 0xf0 && (second & 0xf0) == 0x80 - } -} - -/// Decodes the codepoint represented by a multi-byte UTF-8 sequence. -/// -/// Does not check that the codepoint is valid, -/// and returns `u32` because casting invalid codepoints to `char` is insta UB. -fn merge_nonascii_unchecked_utf8(src: &[u8]) -> u32 { - let mut c = src[0] as u32 & (0x7f >> src.len()); - for b in &src[1..] { - c = (c << 6) | (b & 0b0011_1111) as u32; - } - c -} - -/// Create a `char` from a leading and a trailing surrogate. -/// -/// This function is safe because it ignores the six most significant bits of -/// each arguments and always produces a codepoint in 0x01_00_00..=0x10_ff_ff. -fn combine_surrogates(first: u16, second: u16) -> char { - unsafe { - let high = (first & 0x_03_ff) as u32; - let low = (second & 0x_03_ff) as u32; - let c = ((high << 10) | low) + 0x_01_00_00; // no, the constant can't be or'd in - char::from_u32_unchecked(c) - } -} - - - -/// Adds `.utf8chars()` and `.utf16chars()` iterator constructors to `&str`. -pub trait StrExt: AsRef<str> { - /// Equivalent to `.chars()` but produces `Utf8Char`s. - fn utf8chars(&self) -> Utf8Chars; - /// Equivalent to `.chars()` but produces `Utf16Char`s. - fn utf16chars(&self) -> Utf16Chars; - /// Equivalent to `.char_indices()` but produces `Utf8Char`s. - fn utf8char_indices(&self) -> Utf8CharIndices; - /// Equivalent to `.char_indices()` but produces `Utf16Char`s. - fn utf16char_indices(&self) -> Utf16CharIndices; -} - -impl StrExt for str { - fn utf8chars(&self) -> Utf8Chars { - Utf8Chars::from(self) - } - fn utf16chars(&self) -> Utf16Chars { - Utf16Chars::from(self) - } - fn utf8char_indices(&self) -> Utf8CharIndices { - Utf8CharIndices::from(self) - } - fn utf16char_indices(&self) -> Utf16CharIndices { - Utf16CharIndices::from(self) - } -} - -#[cfg(feature="ascii")] -impl StrExt for AsciiStr { - fn utf8chars(&self) -> Utf8Chars { - Utf8Chars::from(self.as_str()) - } - fn utf16chars(&self) -> Utf16Chars { - Utf16Chars::from(self.as_str()) - } - fn utf8char_indices(&self) -> Utf8CharIndices { - Utf8CharIndices::from(self.as_str()) - } - fn utf16char_indices(&self) -> Utf16CharIndices { - Utf16CharIndices::from(self.as_str()) - } -} - - - -/// Iterator methods that convert between `u8`s and `Utf8Char` or `u16`s and `Utf16Char` -/// -/// All the iterator adapters also accept iterators that produce references of -/// the type they convert from. -pub trait IterExt: Iterator+Sized { - /// Converts an iterator of `Utf8Char`s or `&Utf8Char`s to an iterator of - /// `u8`s. - /// - /// Has the same effect as `.flat_map()` or `.flatten()`, but the returned - /// iterator is ~40% faster. - /// - /// The iterator also implements `Read` - /// (when the `std` feature isn't disabled). - /// Reading will never produce an error, and calls to `.read()` and `.next()` - /// can be mixed. - /// - /// The exact number of bytes cannot be known in advance, but `size_hint()` - /// gives the possible range. - /// (min: all remaining characters are ASCII, max: all require four bytes) - /// - /// # Examples - /// - /// From iterator of values: - /// - /// ``` - /// use encode_unicode::{IterExt, StrExt}; - /// - /// let iterator = "foo".utf8chars(); - /// let mut bytes = [0; 4]; - /// for (u,dst) in iterator.to_bytes().zip(&mut bytes) {*dst=u;} - /// assert_eq!(&bytes, b"foo\0"); - /// ``` - /// - /// From iterator of references: - /// - #[cfg_attr(feature="std", doc=" ```")] - #[cfg_attr(not(feature="std"), doc=" ```no_compile")] - /// use encode_unicode::{IterExt, StrExt, Utf8Char}; - /// - /// let chars: Vec<Utf8Char> = "💣 bomb 💣".utf8chars().collect(); - /// let bytes: Vec<u8> = chars.iter().to_bytes().collect(); - /// let flat_map: Vec<u8> = chars.iter().flat_map(|u8c| *u8c ).collect(); - /// assert_eq!(bytes, flat_map); - /// ``` - /// - /// `Read`ing from it: - /// - #[cfg_attr(feature="std", doc=" ```")] - #[cfg_attr(not(feature="std"), doc=" ```no_compile")] - /// use encode_unicode::{IterExt, StrExt}; - /// use std::io::Read; - /// - /// let s = "Ååh‽"; - /// assert_eq!(s.len(), 8); - /// let mut buf = [b'E'; 9]; - /// let mut reader = s.utf8chars().to_bytes(); - /// assert_eq!(reader.read(&mut buf[..]).unwrap(), 8); - /// assert_eq!(reader.read(&mut buf[..]).unwrap(), 0); - /// assert_eq!(&buf[..8], s.as_bytes()); - /// assert_eq!(buf[8], b'E'); - /// ``` - fn to_bytes(self) -> Utf8CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf8Char>; - - /// Converts an iterator of `Utf16Char` (or `&Utf16Char`) to an iterator of - /// `u16`s. - /// - /// Has the same effect as `.flat_map()` or `.flatten()`, but the returned - /// iterator is about twice as fast. - /// - /// The exact number of units cannot be known in advance, but `size_hint()` - /// gives the possible range. - /// - /// # Examples - /// - /// From iterator of values: - /// - /// ``` - /// use encode_unicode::{IterExt, StrExt}; - /// - /// let iterator = "foo".utf16chars(); - /// let mut units = [0; 4]; - /// for (u,dst) in iterator.to_units().zip(&mut units) {*dst=u;} - /// - /// assert_eq!(units, ['f' as u16, 'o' as u16, 'o' as u16, 0]); - /// ``` - /// - /// From iterator of references: - /// - #[cfg_attr(feature="std", doc=" ```")] - #[cfg_attr(not(feature="std"), doc=" ```no_compile")] - /// use encode_unicode::{IterExt, StrExt, Utf16Char}; - /// - /// // (💣 takes two units) - /// let chars: Vec<Utf16Char> = "💣 bomb 💣".utf16chars().collect(); - /// let units: Vec<u16> = chars.iter().to_units().collect(); - /// let flat_map: Vec<u16> = chars.iter().flat_map(|u16c| *u16c ).collect(); - /// - /// assert_eq!(units, flat_map); - /// ``` - fn to_units(self) -> Utf16CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf16Char>; - - /// Decodes bytes as UTF-8 and groups them into `Utf8Char`s - /// - /// When errors (invalid values or sequences) are encountered, - /// it continues with the byte right after the start of the error sequence. - /// This is neither the most intelligent choiche (sometimes it is guaranteed to - /// produce another error), nor the easiest to implement, but I believe it to - /// be the most predictable. - /// It also means that ASCII characters are never hidden by errors. - /// - /// # Examples - /// - /// Replace all errors with u+FFFD REPLACEMENT_CHARACTER: - /// ``` - /// use encode_unicode::{Utf8Char, IterExt}; - /// - /// let mut buf = [b'\0'; 255]; - /// let len = b"foo\xCFbar".iter() - /// .to_utf8chars() - /// .flat_map(|r| r.unwrap_or(Utf8Char::from('\u{FFFD}')).into_iter() ) - /// .zip(&mut buf[..]) - /// .map(|(byte, dst)| *dst = byte ) - /// .count(); - /// - /// assert_eq!(&buf[..len], "foo\u{FFFD}bar".as_bytes()); - /// ``` - /// - /// Collect everything up until the first error into a string: - #[cfg_attr(feature="std", doc=" ```")] - #[cfg_attr(not(feature="std"), doc=" ```no_compile")] - /// use encode_unicode::iterator::Utf8CharMerger; - /// let mut good = String::new(); - /// for r in Utf8CharMerger::from(b"foo\xcc\xbbbar\xcc\xddbaz") { - /// if let Ok(uc) = r { - /// good.push_str(uc.as_str()); - /// } else { - /// break; - /// } - /// } - /// assert_eq!(good, "foo̻bar"); - /// ``` - /// - /// Abort decoding on error: - #[cfg_attr(feature="std", doc=" ```")] - #[cfg_attr(not(feature="std"), doc=" ```no_compile")] - /// use encode_unicode::{IterExt, Utf8Char}; - /// use encode_unicode::error::{InvalidUtf8Slice, InvalidUtf8}; - /// - /// let result = b"ab\0\xe0\xbc\xa9 \xf3\x80\x77".iter() - /// .to_utf8chars() - /// .collect::<Result<String,InvalidUtf8Slice>>(); - /// - /// assert_eq!(result, Err(InvalidUtf8Slice::Utf8(InvalidUtf8::NotAContinuationByte(2)))); - /// ``` - fn to_utf8chars(self) -> Utf8CharMerger<Self::Item,Self> where Self::Item: Borrow<u8>; - - /// Decodes bytes as UTF-16 and groups them into `Utf16Char`s - /// - /// When errors (unmatched leading surrogates or unexpected trailing surrogates) - /// are encountered, an error is produced for every unit. - /// - /// # Examples - /// - /// Replace errors with '�': - #[cfg_attr(feature="std", doc=" ```")] - #[cfg_attr(not(feature="std"), doc=" ```no_compile")] - /// use encode_unicode::{IterExt, Utf16Char}; - /// - /// let slice = &['a' as u16, 0xdf00, 0xd83c, 0xdca0][..]; - /// let string = slice.iter() - /// .to_utf16chars() - /// .map(|r| r.unwrap_or(Utf16Char::from('\u{fffd}')) ) // REPLACEMENT_CHARACTER - /// .collect::<String>(); - /// - /// assert_eq!(string, "a�🂠"); - /// ``` - /// - /// ``` - /// use encode_unicode::{IterExt, Utf16Char}; - /// use encode_unicode::error::Utf16PairError::*; - /// - /// let slice = [0xdcba, 0xdeff, 0xd8be, 0xdeee, 'Y' as u16, 0xdab1, 0xdab1]; - /// let mut iter = slice.iter().to_utf16chars(); - /// assert_eq!(iter.size_hint(), (3, Some(7))); - /// assert_eq!(iter.next(), Some(Err(UnexpectedTrailingSurrogate))); - /// assert_eq!(iter.next(), Some(Err(UnexpectedTrailingSurrogate))); - /// assert_eq!(iter.next(), Some(Ok(Utf16Char::from('\u{3faee}')))); - /// assert_eq!(iter.next(), Some(Ok(Utf16Char::from('Y')))); - /// assert_eq!(iter.next(), Some(Err(UnmatchedLeadingSurrogate))); - /// assert_eq!(iter.next(), Some(Err(Incomplete))); - /// assert_eq!(iter.into_remaining_units().next(), None); - /// ``` - /// - /// Search for a codepoint and return the codepoint index of the first match: - /// ``` - /// use encode_unicode::{IterExt, Utf16Char}; - /// - /// let position = [0xd875, 0xdd4f, '≈' as u16, '2' as u16].iter() - /// .to_utf16chars() - /// .position(|r| r == Ok(Utf16Char::from('≈')) ); - /// - /// assert_eq!(position, Some(1)); - /// ``` - fn to_utf16chars(self) -> Utf16CharMerger<Self::Item,Self> where Self::Item: Borrow<u16>; -} - -impl<I:Iterator> IterExt for I { - fn to_bytes(self) -> Utf8CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf8Char> { - iter_bytes(self) - } - fn to_units(self) -> Utf16CharSplitter<Self::Item,Self> where Self::Item: Borrow<Utf16Char> { - iter_units(self) - } - fn to_utf8chars(self) -> Utf8CharMerger<Self::Item,Self> where Self::Item: Borrow<u8> { - Utf8CharMerger::from(self) - } - fn to_utf16chars(self) -> Utf16CharMerger<Self::Item,Self> where Self::Item: Borrow<u16> { - Utf16CharMerger::from(self) - } -} - - -/// Methods for iterating over `u8` and `u16` slices as UTF-8 or UTF-16 characters. -/// -/// The iterators are slightly faster than the similar methods in [`IterExt`](trait.IterExt.html) -/// because they con "push back" items for free after errors and don't need a -/// separate buffer that must be checked on every call to `.next()`. -pub trait SliceExt: Index<RangeFull> { - /// Decode `u8` slices as UTF-8 and iterate over the codepoints as `Utf8Char`s, - /// - /// # Examples - /// - /// Get the index and error type of the first error: - #[cfg_attr(feature="std", doc=" ```")] - #[cfg_attr(not(feature="std"), doc=" ```no_compile")] - /// use encode_unicode::{SliceExt, Utf8Char}; - /// use encode_unicode::error::InvalidUtf8Slice; - /// - /// let slice = b"ab\0\xe0\xbc\xa9 \xf3\x80\x77"; - /// let result = slice.utf8char_indices() - /// .map(|(offset,r,length)| r.map_err(|e| (offset,e,length) ) ) - /// .collect::<Result<String,(usize,InvalidUtf8Slice,usize)>>(); - /// - /// assert_eq!(result, Err((7, InvalidUtf8Slice::TooShort(4), 1))); - /// ``` - /// - /// ``` - /// use encode_unicode::{SliceExt, Utf8Char}; - /// use std::error::Error; - /// - /// let slice = b"\xf0\xbf\xbf\xbfXY\xdd\xbb\xe1\x80\x99quux123"; - /// let mut fixed_size = [Utf8Char::default(); 8]; - /// for (cp_i, (byte_index, r, _)) in slice.utf8char_indices().enumerate().take(8) { - /// match r { - /// Ok(u8c) => fixed_size[cp_i] = u8c, - /// Err(e) => panic!("Invalid codepoint at index {} ({})", cp_i, e.description()), - /// } - /// } - /// let chars = ['\u{3ffff}', 'X', 'Y', '\u{77b}', '\u{1019}', 'q', 'u', 'u']; - /// assert_eq!(fixed_size, chars); - /// ``` - /// - #[cfg_attr(feature="std", doc=" ```")] - #[cfg_attr(not(feature="std"), doc=" ```no_compile")] - /// use encode_unicode::{SliceExt, Utf8Char}; - /// use encode_unicode::error::InvalidUtf8Slice::*; - /// use encode_unicode::error::{InvalidUtf8, InvalidUtf8FirstByte, InvalidCodepoint}; - /// - /// let bytes = b"\xfa-\xf4\x8f\xee\xa1\x8f-\xed\xa9\x87\xf0\xcc\xbb"; - /// let mut errors = Vec::new(); - /// let mut lengths = Vec::new(); - /// let mut string = String::new(); - /// for (offset,result,length) in bytes.utf8char_indices() { - /// lengths.push((offset,length)); - /// let c = result.unwrap_or_else(|error| { - /// errors.push((offset,error)); - /// Utf8Char::from('\u{fffd}') // replacement character - /// }); - /// string.push_str(c.as_str()); - /// } - /// - /// assert_eq!(string, "�-��\u{e84f}-����\u{33b}"); - /// assert_eq!(lengths, [(0,1), (1,1), (2,1), (3,1), (4,3), (7,1), - /// (8,1), (9,1), (10,1), (11,1), (12,2)]); - /// assert_eq!(errors, [ - /// ( 0, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::TooLongSeqence))), - /// ( 2, Utf8(InvalidUtf8::NotAContinuationByte(2))), - /// ( 3, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::ContinuationByte))), - /// ( 8, Codepoint(InvalidCodepoint::Utf16Reserved)), - /// ( 9, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::ContinuationByte))), - /// (10, Utf8(InvalidUtf8::FirstByte(InvalidUtf8FirstByte::ContinuationByte))), - /// (11, TooShort(4)), // (but it was not the last element returned!) - /// ]); - /// ``` - fn utf8char_indices(&self) -> Utf8CharDecoder where Self::Output: Borrow<[u8]>; - - - /// Decode `u16` slices as UTF-16 and iterate over the codepoints as `Utf16Char`s, - /// - /// The iterator produces `(usize,Result<Utf16Char,Utf16Error>,usize)`, - /// and the slice is validated as you go. - /// - /// The first `usize` contains the offset from the start of the slice and - /// the last `usize` contains the length of the codepoint or error. - /// The length is either 1 or 2, and always 1 for errors. - /// - /// # Examples - /// - #[cfg_attr(feature="std", doc=" ```")] - #[cfg_attr(not(feature="std"), doc=" ```no_compile")] - /// use encode_unicode::{SliceExt, Utf8Char}; - /// - /// let slice = &['a' as u16, 0xdf00, 0xd83c, 0xdca0][..]; - /// let mut errors = Vec::new(); - /// let string = slice.utf16char_indices().map(|(offset,r,_)| match r { - /// Ok(u16c) => Utf8Char::from(u16c), - /// Err(_) => { - /// errors.push(offset); - /// Utf8Char::from('\u{fffd}') // REPLACEMENT_CHARACTER - /// } - /// }).collect::<String>(); - /// - /// assert_eq!(string, "a�🂠"); - /// assert_eq!(errors, [1]); - /// ``` - /// - /// Search for a codepoint and return its unit and codepoint index. - /// ``` - /// use encode_unicode::{SliceExt, Utf16Char}; - /// - /// let slice = [0xd875,/*'𝕏'*/ 0xdd4f, '≈' as u16, '2' as u16]; - /// let position = slice.utf16char_indices() - /// .enumerate() - /// .find(|&(_,(_,r,_))| r == Ok(Utf16Char::from('≈')) ) - /// .map(|(codepoint, (offset, _, _))| (codepoint, offset) ); - /// - /// assert_eq!(position, Some((1,2))); - /// ``` - /// - /// Error types: - /// ``` - /// use encode_unicode::{SliceExt, Utf16Char}; - /// use encode_unicode::error::Utf16PairError::*; - /// - /// let slice = [0xdcba, 0xdeff, 0xd8be, 0xdeee, 'λ' as u16, 0xdab1, 0xdab1]; - /// let mut iter = slice.utf16char_indices(); - /// assert_eq!(iter.next(), Some((0, Err(UnexpectedTrailingSurrogate), 1))); - /// assert_eq!(iter.next(), Some((1, Err(UnexpectedTrailingSurrogate), 1))); - /// assert_eq!(iter.next(), Some((2, Ok(Utf16Char::from('\u{3faee}')), 2))); - /// assert_eq!(iter.next(), Some((4, Ok(Utf16Char::from('λ')), 1))); - /// assert_eq!(iter.next(), Some((5, Err(UnmatchedLeadingSurrogate), 1))); - /// assert_eq!(iter.next(), Some((6, Err(Incomplete), 1))); - /// assert_eq!(iter.next(), None); - /// assert_eq!(iter.as_slice(), []) - /// ``` - fn utf16char_indices(&self) -> Utf16CharDecoder where Self::Output: Borrow<[u16]>; -} - -impl<S: ?Sized+Index<RangeFull>> SliceExt for S { - fn utf8char_indices(&self) -> Utf8CharDecoder where Self::Output: Borrow<[u8]> { - Utf8CharDecoder::from(self[..].borrow()) - } - fn utf16char_indices(&self) -> Utf16CharDecoder where Self::Output: Borrow<[u16]> { - Utf16CharDecoder::from(self[..].borrow()) - } -} diff --git a/vendor/encode_unicode/src/utf16_char.rs b/vendor/encode_unicode/src/utf16_char.rs deleted file mode 100644 index d8a6cec..0000000 --- a/vendor/encode_unicode/src/utf16_char.rs +++ /dev/null @@ -1,687 +0,0 @@ -/* Copyright 2016 The encode_unicode Developers - * - * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or - * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or - * http://opensource.org/licenses/MIT>, at your option. This file may not be - * copied, modified, or distributed except according to those terms. - */ - -use utf16_iterators::Utf16Iterator; -use traits::{CharExt, U16UtfExt}; -use utf8_char::Utf8Char; -use errors::{InvalidUtf16Slice, InvalidUtf16Array, InvalidUtf16Tuple}; -use errors::{NonBMPError, EmptyStrError, FromStrError}; -extern crate core; -use self::core::{hash,fmt}; -use self::core::cmp::Ordering; -use self::core::borrow::Borrow; -use self::core::ops::Deref; -use self::core::str::FromStr; -#[cfg(feature="std")] -use self::core::iter::FromIterator; -#[cfg(feature="std")] -#[allow(deprecated)] -use std::ascii::AsciiExt; -#[cfg(feature="ascii")] -use self::core::char; -#[cfg(feature="ascii")] -extern crate ascii; -#[cfg(feature="ascii")] -use self::ascii::{AsciiChar,ToAsciiChar,ToAsciiCharError}; - - -// I don't think there is any good default value for char, but char does. -#[derive(Default)] -// char doesn't do anything more advanced than u32 for Eq/Ord, so we shouldn't either. -// When it's a single unit, the second is zero, so Eq works. -// #[derive(Ord)] however, breaks on surrogate pairs. -#[derive(PartialEq,Eq)] -#[derive(Clone,Copy)] - - -/// An unicode codepoint stored as UTF-16. -/// -/// It can be borrowed as an `u16` slice, and has the same size as `char`. -pub struct Utf16Char { - units: [u16; 2], -} - - - ///////////////////// - //conversion traits// -///////////////////// -impl FromStr for Utf16Char { - type Err = FromStrError; - /// Create an `Utf16Char` from a string slice. - /// The string must contain exactly one codepoint. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::error::FromStrError::*; - /// use encode_unicode::Utf16Char; - /// use std::str::FromStr; - /// - /// assert_eq!(Utf16Char::from_str("a"), Ok(Utf16Char::from('a'))); - /// assert_eq!(Utf16Char::from_str("🂠"), Ok(Utf16Char::from('🂠'))); - /// assert_eq!(Utf16Char::from_str(""), Err(Empty)); - /// assert_eq!(Utf16Char::from_str("ab"), Err(MultipleCodepoints)); - /// assert_eq!(Utf16Char::from_str("é"), Err(MultipleCodepoints));// 'e'+u301 combining mark - /// ``` - fn from_str(s: &str) -> Result<Self, FromStrError> { - match Utf16Char::from_str_start(s) { - Ok((u16c,bytes)) if bytes == s.len() => Ok(u16c), - Ok((_,_)) => Err(FromStrError::MultipleCodepoints), - Err(EmptyStrError) => Err(FromStrError::Empty), - } - } -} -impl From<char> for Utf16Char { - fn from(c: char) -> Self { - let (first, second) = c.to_utf16_tuple(); - Utf16Char{ units: [first, second.unwrap_or(0)] } - } -} -impl From<Utf8Char> for Utf16Char { - fn from(utf8: Utf8Char) -> Utf16Char { - let (b, utf8_len) = utf8.to_array(); - match utf8_len { - 1 => Utf16Char{ units: [b[0] as u16, 0] }, - 4 => {// need surrogate - let mut first = 0xd800 - (0x01_00_00u32 >> 10) as u16; - first += (b[0] as u16 & 0x07) << 8; - first += (b[1] as u16 & 0x3f) << 2; - first += (b[2] as u16 & 0x30) >> 4; - let mut second = 0xdc00; - second |= (b[2] as u16 & 0x0f) << 6; - second |= b[3] as u16 & 0x3f; - Utf16Char{ units: [first, second] } - }, - _ => { // 2 or 3 - let mut unit = ((b[0] as u16 & 0x1f) << 6) | (b[1] as u16 & 0x3f); - if utf8_len == 3 { - unit = (unit << 6) | (b[2] as u16 & 0x3f); - } - Utf16Char{ units: [unit, 0] } - }, - } - } -} -impl From<Utf16Char> for char { - fn from(uc: Utf16Char) -> char { - char::from_utf16_array_unchecked(uc.to_array()) - } -} -impl IntoIterator for Utf16Char { - type Item=u16; - type IntoIter=Utf16Iterator; - /// Iterate over the units. - fn into_iter(self) -> Utf16Iterator { - Utf16Iterator::from(self) - } -} - -#[cfg(feature="std")] -impl Extend<Utf16Char> for Vec<u16> { - fn extend<I:IntoIterator<Item=Utf16Char>>(&mut self, iter: I) { - let iter = iter.into_iter(); - self.reserve(iter.size_hint().0); - for u16c in iter { - self.push(u16c.units[0]); - if u16c.units[1] != 0 { - self.push(u16c.units[1]); - } - } - } -} -#[cfg(feature="std")] -impl<'a> Extend<&'a Utf16Char> for Vec<u16> { - fn extend<I:IntoIterator<Item=&'a Utf16Char>>(&mut self, iter: I) { - self.extend(iter.into_iter().cloned()) - } -} -#[cfg(feature="std")] -impl FromIterator<Utf16Char> for Vec<u16> { - fn from_iter<I:IntoIterator<Item=Utf16Char>>(iter: I) -> Self { - let mut vec = Vec::new(); - vec.extend(iter); - return vec; - } -} -#[cfg(feature="std")] -impl<'a> FromIterator<&'a Utf16Char> for Vec<u16> { - fn from_iter<I:IntoIterator<Item=&'a Utf16Char>>(iter: I) -> Self { - Self::from_iter(iter.into_iter().cloned()) - } -} - -#[cfg(feature="std")] -impl Extend<Utf16Char> for String { - fn extend<I:IntoIterator<Item=Utf16Char>>(&mut self, iter: I) { - self.extend(iter.into_iter().map(|u16c| Utf8Char::from(u16c) )); - } -} -#[cfg(feature="std")] -impl<'a> Extend<&'a Utf16Char> for String { - fn extend<I:IntoIterator<Item=&'a Utf16Char>>(&mut self, iter: I) { - self.extend(iter.into_iter().cloned()); - } -} -#[cfg(feature="std")] -impl FromIterator<Utf16Char> for String { - fn from_iter<I:IntoIterator<Item=Utf16Char>>(iter: I) -> Self { - let mut s = String::new(); - s.extend(iter); - return s; - } -} -#[cfg(feature="std")] -impl<'a> FromIterator<&'a Utf16Char> for String { - fn from_iter<I:IntoIterator<Item=&'a Utf16Char>>(iter: I) -> Self { - Self::from_iter(iter.into_iter().cloned()) - } -} - - - ///////////////// - //getter traits// -///////////////// -impl AsRef<[u16]> for Utf16Char { - #[inline] - fn as_ref(&self) -> &[u16] { - &self.units[..self.len()] - } -} -impl Borrow<[u16]> for Utf16Char { - #[inline] - fn borrow(&self) -> &[u16] { - self.as_ref() - } -} -impl Deref for Utf16Char { - type Target = [u16]; - #[inline] - fn deref(&self) -> &[u16] { - self.as_ref() - } -} - - - //////////////// - //ascii traits// -//////////////// -#[cfg(feature="std")] -#[allow(deprecated)] -impl AsciiExt for Utf16Char { - type Owned = Self; - fn is_ascii(&self) -> bool { - self.units[0] < 128 - } - fn eq_ignore_ascii_case(&self, other: &Self) -> bool { - self.to_ascii_lowercase() == other.to_ascii_lowercase() - } - fn to_ascii_uppercase(&self) -> Self { - let n = self.units[0].wrapping_sub(b'a' as u16); - if n < 26 {Utf16Char{ units: [n+b'A' as u16, 0] }} - else {*self} - } - fn to_ascii_lowercase(&self) -> Self { - let n = self.units[0].wrapping_sub(b'A' as u16); - if n < 26 {Utf16Char{ units: [n+b'a' as u16, 0] }} - else {*self} - } - fn make_ascii_uppercase(&mut self) { - *self = self.to_ascii_uppercase() - } - fn make_ascii_lowercase(&mut self) { - *self = self.to_ascii_lowercase(); - } -} - -#[cfg(feature="ascii")] -/// Requires the feature "ascii". -impl From<AsciiChar> for Utf16Char { - #[inline] - fn from(ac: AsciiChar) -> Self { - Utf16Char{ units: [ac.as_byte() as u16, 0] } - } -} -#[cfg(feature="ascii")] -/// Requires the feature "ascii". -impl ToAsciiChar for Utf16Char { - #[inline] - fn to_ascii_char(self) -> Result<AsciiChar, ToAsciiCharError> { - // ToAsciiCHar is not implemented for u16 in ascii 0.9.0 - if self.is_ascii() {self.units[0] as u8} else {255}.to_ascii_char() - } - #[inline] - unsafe fn to_ascii_char_unchecked(self) -> AsciiChar { - (self.units[0] as u8).to_ascii_char_unchecked() - } -} - - - ///////////////////////////////////////////////////////// - //Genaral traits that cannot be derived to emulate char// -///////////////////////////////////////////////////////// -impl hash::Hash for Utf16Char { - fn hash<H : hash::Hasher>(&self, state: &mut H) { - self.to_char().hash(state); - } -} -impl fmt::Debug for Utf16Char { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self.to_char(), fmtr) - } -} -impl fmt::Display for Utf16Char { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&Utf8Char::from(*self), fmtr) - } -} -// Cannot derive these impls because two-unit characters must always compare -// greater than one-unit ones. -impl PartialOrd for Utf16Char { - #[inline] - fn partial_cmp(&self, rhs: &Self) -> Option<Ordering> { - Some(self.cmp(rhs)) - } -} -impl Ord for Utf16Char { - #[inline] - fn cmp(&self, rhs: &Self) -> Ordering { - // Shift the first unit by 0xd if surrogate, and 0 otherwise. - // This ensures surrogates are always greater than 0xffff, and - // that the second unit only affect the result when the first are equal. - // Multiplying by a constant factor isn't enough because that factor - // would have to be greater than 1023 and smaller than 5.5. - // This transformation is less complicated than combine_surrogates(). - let lhs = (self.units[0] as u32, self.units[1] as u32); - let rhs = (rhs.units[0] as u32, rhs.units[1] as u32); - let lhs = (lhs.0 << (lhs.1 >> 12)) + lhs.1; - let rhs = (rhs.0 << (rhs.1 >> 12)) + rhs.1; - lhs.cmp(&rhs) - } -} - - - //////////////////////////////// - //Comparisons with other types// -//////////////////////////////// -impl PartialEq<char> for Utf16Char { - fn eq(&self, u32c: &char) -> bool { - *self == Utf16Char::from(*u32c) - } -} -impl PartialEq<Utf16Char> for char { - fn eq(&self, u16c: &Utf16Char) -> bool { - Utf16Char::from(*self) == *u16c - } -} -impl PartialOrd<char> for Utf16Char { - fn partial_cmp(&self, u32c: &char) -> Option<Ordering> { - self.partial_cmp(&Utf16Char::from(*u32c)) - } -} -impl PartialOrd<Utf16Char> for char { - fn partial_cmp(&self, u16c: &Utf16Char) -> Option<Ordering> { - Utf16Char::from(*self).partial_cmp(u16c) - } -} - -impl PartialEq<Utf8Char> for Utf16Char { - fn eq(&self, u8c: &Utf8Char) -> bool { - *self == Utf16Char::from(*u8c) - } -} -impl PartialOrd<Utf8Char> for Utf16Char { - fn partial_cmp(&self, u8c: &Utf8Char) -> Option<Ordering> { - self.partial_cmp(&Utf16Char::from(*u8c)) - } -} -// The other direction is implemented in utf8_char.rs - -/// Only considers the unit equal if the codepoint of the `Utf16Char` is not -/// made up of a surrogate pair. -/// -/// There is no impl in the opposite direction, as this should only be used to -/// compare `Utf16Char`s against constants. -/// -/// # Examples -/// -/// ``` -/// # use encode_unicode::Utf16Char; -/// assert!(Utf16Char::from('6') == b'6' as u16); -/// assert!(Utf16Char::from('\u{FFFF}') == 0xffff_u16); -/// assert!(Utf16Char::from_tuple((0xd876, Some(0xdef9))).unwrap() != 0xd876_u16); -/// ``` -impl PartialEq<u16> for Utf16Char { - fn eq(&self, unit: &u16) -> bool { - self.units[0] == *unit && self.units[1] == 0 - } -} -/// Only considers the byte equal if the codepoint of the `Utf16Char` is <= U+FF. -/// -/// # Examples -/// -/// ``` -/// # use encode_unicode::Utf16Char; -/// assert!(Utf16Char::from('6') == b'6'); -/// assert!(Utf16Char::from('\u{00FF}') == b'\xff'); -/// assert!(Utf16Char::from('\u{0100}') != b'\0'); -/// ``` -impl PartialEq<u8> for Utf16Char { - fn eq(&self, byte: &u8) -> bool { - self.units[0] == *byte as u16 - } -} -#[cfg(feature = "ascii")] -/// `Utf16Char`s that are not ASCII never compare equal. -impl PartialEq<AsciiChar> for Utf16Char { - #[inline] - fn eq(&self, ascii: &AsciiChar) -> bool { - self.units[0] == *ascii as u16 - } -} -#[cfg(feature = "ascii")] -/// `Utf16Char`s that are not ASCII never compare equal. -impl PartialEq<Utf16Char> for AsciiChar { - #[inline] - fn eq(&self, u16c: &Utf16Char) -> bool { - *self as u16 == u16c.units[0] - } -} -#[cfg(feature = "ascii")] -/// `Utf16Char`s that are not ASCII always compare greater. -impl PartialOrd<AsciiChar> for Utf16Char { - #[inline] - fn partial_cmp(&self, ascii: &AsciiChar) -> Option<Ordering> { - self.units[0].partial_cmp(&(*ascii as u16)) - } -} -#[cfg(feature = "ascii")] -/// `Utf16Char`s that are not ASCII always compare greater. -impl PartialOrd<Utf16Char> for AsciiChar { - #[inline] - fn partial_cmp(&self, u16c: &Utf16Char) -> Option<Ordering> { - (*self as u16).partial_cmp(&u16c.units[0]) - } -} - - - /////////////////////////////////////////////////////// - //pub impls that should be together for nicer rustdoc// -/////////////////////////////////////////////////////// -impl Utf16Char { - /// Create an `Utf16Char` from the first codepoint in a string slice, - /// converting from UTF-8 to UTF-16. - /// - /// The returned `usize` is the number of UTF-8 bytes used from the str, - /// and not the number of UTF-16 units. - /// - /// Returns an error if the `str` is empty. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::Utf16Char; - /// - /// assert_eq!(Utf16Char::from_str_start("a"), Ok((Utf16Char::from('a'),1))); - /// assert_eq!(Utf16Char::from_str_start("ab"), Ok((Utf16Char::from('a'),1))); - /// assert_eq!(Utf16Char::from_str_start("🂠 "), Ok((Utf16Char::from('🂠'),4))); - /// assert_eq!(Utf16Char::from_str_start("é"), Ok((Utf16Char::from('e'),1)));// 'e'+u301 combining mark - /// assert!(Utf16Char::from_str_start("").is_err()); - /// ``` - pub fn from_str_start(s: &str) -> Result<(Self,usize), EmptyStrError> { - if s.is_empty() { - return Err(EmptyStrError); - } - let b = s.as_bytes(); - // Read the last byte first to reduce the number of unnecesary length checks. - match b[0] { - 0...127 => {// 1 byte => 1 unit - let unit = b[0] as u16;// 0b0000_0000_0xxx_xxxx - Ok((Utf16Char{ units: [unit, 0] }, 1)) - }, - 0b1000_0000...0b1101_1111 => {// 2 bytes => 1 unit - let unit = (((b[1] & 0x3f) as u16) << 0) // 0b0000_0000_00xx_xxxx - | (((b[0] & 0x1f) as u16) << 6);// 0b0000_0xxx_xx00_0000 - Ok((Utf16Char{ units: [unit, 0] }, 2)) - }, - 0b1110_0000...0b1110_1111 => {// 3 bytes => 1 unit - let unit = (((b[2] & 0x3f) as u16) << 0) // 0b0000_0000_00xx_xxxx - | (((b[1] & 0x3f) as u16) << 6) // 0b0000_xxxx_xx00_0000 - | (((b[0] & 0x0f) as u16) << 12);// 0bxxxx_0000_0000_0000 - Ok((Utf16Char{ units: [unit, 0] }, 3)) - }, - _ => {// 4 bytes => 2 units - let second = 0xdc00 // 0b1101_1100_0000_0000 - | (((b[3] & 0x3f) as u16) << 0) // 0b0000_0000_00xx_xxxx - | (((b[2] & 0x0f) as u16) << 6);// 0b0000_00xx_xx00_0000 - let first = 0xd800-(0x01_00_00u32>>10) as u16// 0b1101_0111_1100_0000 - + (((b[2] & 0x30) as u16) >> 4) // 0b0000_0000_0000_00xx - + (((b[1] & 0x3f) as u16) << 2) // 0b0000_0000_xxxx_xx00 - + (((b[0] & 0x07) as u16) << 8); // 0b0000_0xxx_0000_0000 - Ok((Utf16Char{ units: [first, second] }, 4)) - } - } - } - /// Validate and store the first UTF-16 codepoint in the slice. - /// Also return how many units were needed. - pub fn from_slice_start(src: &[u16]) -> Result<(Self,usize), InvalidUtf16Slice> { - char::from_utf16_slice_start(src).map(|(_,len)| { - let second = if len==2 {src[1]} else {0}; - (Utf16Char{ units: [src[0], second] }, len) - }) - } - /// Store the first UTF-16 codepoint of the slice. - /// - /// # Safety - /// - /// The slice must be non-empty and start with a valid UTF-16 codepoint. - /// The length of the slice is never checked. - pub unsafe fn from_slice_start_unchecked(src: &[u16]) -> (Self,usize) { - let first = *src.get_unchecked(0); - if first.is_utf16_leading_surrogate() { - (Utf16Char{ units: [first, *src.get_unchecked(1)] }, 2) - } else { - (Utf16Char{ units: [first, 0] }, 1) - } - } - /// Validate and store an UTF-16 array as returned from `char.to_utf16_array()`. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::Utf16Char; - /// use encode_unicode::error::InvalidUtf16Array; - /// - /// assert_eq!(Utf16Char::from_array(['x' as u16, 'y' as u16]), Ok(Utf16Char::from('x'))); - /// assert_eq!(Utf16Char::from_array(['睷' as u16, 0]), Ok(Utf16Char::from('睷'))); - /// assert_eq!(Utf16Char::from_array([0xda6f, 0xdcde]), Ok(Utf16Char::from('\u{abcde}'))); - /// assert_eq!(Utf16Char::from_array([0xf111, 0xdbad]), Ok(Utf16Char::from('\u{f111}'))); - /// assert_eq!(Utf16Char::from_array([0xdaaf, 0xdaaf]), Err(InvalidUtf16Array::SecondIsNotTrailingSurrogate)); - /// assert_eq!(Utf16Char::from_array([0xdcac, 0x9000]), Err(InvalidUtf16Array::FirstIsTrailingSurrogate)); - /// ``` - pub fn from_array(units: [u16; 2]) -> Result<Self,InvalidUtf16Array> { - if (units[0] & 0xf8_00) != 0xd8_00 { - Ok(Utf16Char { units: [units[0], 0] }) - } else if units[0] < 0xdc_00 && (units[1] & 0xfc_00) == 0xdc_00 { - Ok(Utf16Char { units: units }) - } else if units[0] < 0xdc_00 { - Err(InvalidUtf16Array::SecondIsNotTrailingSurrogate) - } else { - Err(InvalidUtf16Array::FirstIsTrailingSurrogate) - } - } - /// Create an `Utf16Char` from an array as returned from `char.to_utf16_array()`. - /// - /// # Safety - /// - /// The units must form a valid codepoint, and the second unit must be 0 - /// when a surrogate pair is not required. - /// Violating this can easily lead to undefined behavior, although unlike - /// `char` bad `Utf16Char`s simply existing is not immediately UB. - pub unsafe fn from_array_unchecked(units: [u16; 2]) -> Self { - Utf16Char { units: units } - } - /// Validate and store a UTF-16 pair as returned from `char.to_utf16_tuple()`. - pub fn from_tuple(utf16: (u16,Option<u16>)) -> Result<Self,InvalidUtf16Tuple> { - unsafe {char::from_utf16_tuple(utf16).map(|_| - Self::from_tuple_unchecked(utf16) - )} - } - /// Create an `Utf16Char` from a tuple as returned from `char.to_utf16_tuple()`. - /// - /// # Safety - /// - /// The units must form a valid codepoint with the second being 0 when a - /// surrogate pair is not required. - /// Violating this can easily lead to undefined behavior. - pub unsafe fn from_tuple_unchecked(utf16: (u16,Option<u16>)) -> Self { - Utf16Char { units: [utf16.0, utf16.1.unwrap_or(0)] } - } - /// Create an `Utf16Char` from a single unit. - /// - /// Codepoints < '\u{1_0000}' (which fit in a `u16`) are part of the basic - /// multilingual plane unless they are reserved for surrogate pairs. - /// - /// # Errors - /// - /// Returns `NonBMPError` if the unit is in the range `0xd800..0xe000` - /// (which means that it's part of a surrogat pair) - /// - /// # Examples - /// - /// ``` - /// # use encode_unicode::Utf16Char; - /// assert_eq!(Utf16Char::from_bmp(0x40).unwrap(), '@'); - /// assert_eq!(Utf16Char::from_bmp('ø' as u16).unwrap(), 'ø'); - /// assert!(Utf16Char::from_bmp(0xdddd).is_err()); - /// ``` - pub fn from_bmp(bmp_codepoint: u16) -> Result<Self,NonBMPError> { - if bmp_codepoint & 0xf800 != 0xd800 { - Ok(Utf16Char{ units: [bmp_codepoint, 0] }) - } else { - Err(NonBMPError) - } - } - /// Create an `Utf16Char` from a single unit without checking that it's a - /// valid codepoint on its own. - /// - /// # Safety - /// - /// The unit must be less than 0xd800 or greater than 0xdfff. - /// In other words, not part of a surrogate pair. - /// Violating this can easily lead to undefined behavior. - #[inline] - pub unsafe fn from_bmp_unchecked(bmp_codepoint: u16) -> Self { - Utf16Char{ units: [bmp_codepoint, 0] } - } - /// Checks that the codepoint is in the basic multilingual plane. - /// - /// # Examples - /// ``` - /// # use encode_unicode::Utf16Char; - /// assert_eq!(Utf16Char::from('e').is_bmp(), true); - /// assert_eq!(Utf16Char::from('€').is_bmp(), true); - /// assert_eq!(Utf16Char::from('𝔼').is_bmp(), false); - /// ``` - #[inline] - pub fn is_bmp(&self) -> bool { - self.units[1] == 0 - } - - /// The number of units this character is made up of. - /// - /// Is either 1 or 2 and identical to `.as_char().len_utf16()` - /// or `.as_ref().len()`. - #[inline] - pub fn len(self) -> usize { - 1 + (self.units[1] as usize >> 15) - } - // There is no `.is_emty()` because it would always return false. - - /// Checks that the codepoint is an ASCII character. - #[inline] - pub fn is_ascii(&self) -> bool { - self.units[0] <= 127 - } - /// Checks that two characters are an ASCII case-insensitive match. - /// - /// Is equivalent to `a.to_ascii_lowercase() == b.to_ascii_lowercase()`. - #[cfg(feature="std")] - pub fn eq_ignore_ascii_case(&self, other: &Self) -> bool { - self.to_ascii_lowercase() == other.to_ascii_lowercase() - } - /// Converts the character to its ASCII upper case equivalent. - /// - /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', - /// but non-ASCII letters are unchanged. - #[cfg(feature="std")] - pub fn to_ascii_uppercase(&self) -> Self { - let n = self.units[0].wrapping_sub(b'a' as u16); - if n < 26 {Utf16Char{ units: [n+b'A' as u16, 0] }} - else {*self} - } - /// Converts the character to its ASCII lower case equivalent. - /// - /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', - /// but non-ASCII letters are unchanged. - #[cfg(feature="std")] - pub fn to_ascii_lowercase(&self) -> Self { - let n = self.units[0].wrapping_sub(b'A' as u16); - if n < 26 {Utf16Char{ units: [n+b'a' as u16, 0] }} - else {*self} - } - /// Converts the character to its ASCII upper case equivalent in-place. - /// - /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', - /// but non-ASCII letters are unchanged. - #[cfg(feature="std")] - pub fn make_ascii_uppercase(&mut self) { - *self = self.to_ascii_uppercase() - } - /// Converts the character to its ASCII lower case equivalent in-place. - /// - /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', - /// but non-ASCII letters are unchanged. - #[cfg(feature="std")] - pub fn make_ascii_lowercase(&mut self) { - *self = self.to_ascii_lowercase(); - } - - /// Convert from UTF-16 to UTF-32 - pub fn to_char(self) -> char { - self.into() - } - /// Write the internal representation to a slice, - /// and then returns the number of `u16`s written. - /// - /// # Panics - /// Will panic the buffer is too small; - /// You can get the required length from `.len()`, - /// but a buffer of length two is always large enough. - pub fn to_slice(self, dst: &mut[u16]) -> usize { - // Write the last unit first to avoid repeated length checks. - let extra = self.units[1] as usize >> 15; - match dst.get_mut(extra) { - Some(first) => *first = self.units[extra], - None => panic!("The provided buffer is too small.") - } - if extra != 0 {dst[0] = self.units[0];} - extra+1 - } - /// Get the character represented as an array of two units. - /// - /// The second `u16` is zero for codepoints that fit in one unit. - #[inline] - pub fn to_array(self) -> [u16;2] { - self.units - } - /// The second `u16` is used for surrogate pairs. - #[inline] - pub fn to_tuple(self) -> (u16,Option<u16>) { - (self.units[0], if self.units[1]==0 {None} else {Some(self.units[1])}) - } -} diff --git a/vendor/encode_unicode/src/utf16_iterators.rs b/vendor/encode_unicode/src/utf16_iterators.rs deleted file mode 100644 index 7adb5ac..0000000 --- a/vendor/encode_unicode/src/utf16_iterators.rs +++ /dev/null @@ -1,270 +0,0 @@ -/* Copyright 2016 The encode_unicode Developers - * - * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or - * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or - * http://opensource.org/licenses/MIT>, at your option. This file may not be - * copied, modified, or distributed except according to those terms. - */ - -use traits::CharExt; -use utf16_char::Utf16Char; -use errors::EmptyStrError; -extern crate core; -use self::core::fmt; -use self::core::borrow::Borrow; - -// Invalid values that says the field is consumed or empty. -const FIRST_USED: u16 = 0x_dc_00; -const SECOND_USED: u16 = 0; - -/// Iterate over the units of the UTF-16 representation of a codepoint. -#[derive(Clone)] -pub struct Utf16Iterator { - first: u16, - second: u16, -} -impl From<char> for Utf16Iterator { - fn from(c: char) -> Self { - let (first, second) = c.to_utf16_tuple(); - Utf16Iterator{ first: first, second: second.unwrap_or(SECOND_USED) } - } -} -impl From<Utf16Char> for Utf16Iterator { - fn from(uc: Utf16Char) -> Self { - let (first, second) = uc.to_tuple(); - Utf16Iterator{ first: first, second: second.unwrap_or(SECOND_USED) } - } -} -impl Iterator for Utf16Iterator { - type Item=u16; - fn next(&mut self) -> Option<u16> { - match (self.first, self.second) { - (FIRST_USED, SECOND_USED) => { None }, - (FIRST_USED, second ) => {self.second = SECOND_USED; Some(second)}, - (first , _ ) => {self.first = FIRST_USED; Some(first )}, - } - } - fn size_hint(&self) -> (usize, Option<usize>) { - (self.len(), Some(self.len())) - } -} -impl ExactSizeIterator for Utf16Iterator { - fn len(&self) -> usize { - (if self.first == FIRST_USED {0} else {1}) + - (if self.second == SECOND_USED {0} else {1}) - } -} -impl fmt::Debug for Utf16Iterator { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - let mut clone = self.clone(); - match (clone.next(), clone.next()) { - (Some(one), None) => write!(fmtr, "[{}]", one), - (Some(a), Some(b)) => write!(fmtr, "[{}, {}]", a, b), - (None, _) => write!(fmtr, "[]"), - } - } -} - - - -/// Converts an iterator of `Utf16Char` (or `&Utf16Char`) -/// to an iterator of `u16`s. -/// Is equivalent to calling `.flat_map()` on the original iterator, -/// but the returned iterator is about twice as fast. -/// -/// The exact number of units cannot be known in advance, but `size_hint()` -/// gives the possible range. -/// -/// # Examples -/// -/// From iterator of values: -/// -/// ``` -/// use encode_unicode::{iter_units, CharExt}; -/// -/// let iterator = "foo".chars().map(|c| c.to_utf16() ); -/// let mut units = [0; 4]; -/// for (u,dst) in iter_units(iterator).zip(&mut units) {*dst=u;} -/// assert_eq!(units, ['f' as u16, 'o' as u16, 'o' as u16, 0]); -/// ``` -/// -/// From iterator of references: -/// -#[cfg_attr(feature="std", doc=" ```")] -#[cfg_attr(not(feature="std"), doc=" ```no_compile")] -/// use encode_unicode::{iter_units, CharExt, Utf16Char}; -/// -/// // (💣 takes two units) -/// let chars: Vec<Utf16Char> = "💣 bomb 💣".chars().map(|c| c.to_utf16() ).collect(); -/// let units: Vec<u16> = iter_units(&chars).collect(); -/// let flat_map: Vec<u16> = chars.iter().flat_map(|u16c| *u16c ).collect(); -/// assert_eq!(units, flat_map); -/// ``` -pub fn iter_units<U:Borrow<Utf16Char>, I:IntoIterator<Item=U>> -(iterable: I) -> Utf16CharSplitter<U, I::IntoIter> { - Utf16CharSplitter{ inner: iterable.into_iter(), prev_second: 0 } -} - -/// The iterator type returned by `iter_units()` -#[derive(Clone)] -pub struct Utf16CharSplitter<U:Borrow<Utf16Char>, I:Iterator<Item=U>> { - inner: I, - prev_second: u16, -} -impl<I:Iterator<Item=Utf16Char>> From<I> for Utf16CharSplitter<Utf16Char,I> { - /// A less generic constructor than `iter_units()` - fn from(iter: I) -> Self { - iter_units(iter) - } -} -impl<U:Borrow<Utf16Char>, I:Iterator<Item=U>> Utf16CharSplitter<U,I> { - /// Extracts the source iterator. - /// - /// Note that `iter_units(iter.into_inner())` is not a no-op: - /// If the last returned unit from `next()` was a leading surrogate, - /// the trailing surrogate is lost. - pub fn into_inner(self) -> I { - self.inner - } -} -impl<U:Borrow<Utf16Char>, I:Iterator<Item=U>> Iterator for Utf16CharSplitter<U,I> { - type Item = u16; - fn next(&mut self) -> Option<Self::Item> { - if self.prev_second == 0 { - self.inner.next().map(|u16c| { - let units = u16c.borrow().to_array(); - self.prev_second = units[1]; - units[0] - }) - } else { - let prev_second = self.prev_second; - self.prev_second = 0; - Some(prev_second) - } - } - fn size_hint(&self) -> (usize,Option<usize>) { - // Doesn't need to handle unlikely overflows correctly because - // size_hint() cannot be relied upon anyway. (the trait isn't unsafe) - let (min, max) = self.inner.size_hint(); - let add = if self.prev_second == 0 {0} else {1}; - (min.wrapping_add(add), max.map(|max| max.wrapping_mul(2).wrapping_add(add) )) - } -} - - - -/// An iterator over the codepoints in a `str` represented as `Utf16Char`. -#[derive(Clone)] -pub struct Utf16CharIndices<'a>{ - str: &'a str, - index: usize, -} -impl<'a> From<&'a str> for Utf16CharIndices<'a> { - fn from(s: &str) -> Utf16CharIndices { - Utf16CharIndices{str: s, index: 0} - } -} -impl<'a> Utf16CharIndices<'a> { - /// Extract the remainder of the source `str`. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::{StrExt, Utf16Char}; - /// let mut iter = "abc".utf16char_indices(); - /// assert_eq!(iter.next_back(), Some((2, Utf16Char::from('c')))); - /// assert_eq!(iter.next(), Some((0, Utf16Char::from('a')))); - /// assert_eq!(iter.as_str(), "b"); - /// ``` - pub fn as_str(&self) -> &'a str { - &self.str[self.index..] - } -} -impl<'a> Iterator for Utf16CharIndices<'a> { - type Item = (usize,Utf16Char); - fn next(&mut self) -> Option<(usize,Utf16Char)> { - match Utf16Char::from_str_start(&self.str[self.index..]) { - Ok((u16c, bytes)) => { - let item = (self.index, u16c); - self.index += bytes; - Some(item) - }, - Err(EmptyStrError) => None - } - } - fn size_hint(&self) -> (usize,Option<usize>) { - let len = self.str.len() - self.index; - // For len+3 to overflow, the slice must fill all but two bytes of - // addressable memory, and size_hint() doesn't need to be correct. - (len.wrapping_add(3)/4, Some(len)) - } -} -impl<'a> DoubleEndedIterator for Utf16CharIndices<'a> { - fn next_back(&mut self) -> Option<(usize,Utf16Char)> { - if self.index < self.str.len() { - let rev = self.str.bytes().rev(); - let len = 1 + rev.take_while(|b| b & 0b1100_0000 == 0b1000_0000 ).count(); - let starts = self.str.len() - len; - let (u16c,_) = Utf16Char::from_str_start(&self.str[starts..]).unwrap(); - self.str = &self.str[..starts]; - Some((starts, u16c)) - } else { - None - } - } -} -impl<'a> fmt::Debug for Utf16CharIndices<'a> { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - fmtr.debug_tuple("Utf16CharIndices") - .field(&self.index) - .field(&self.as_str()) - .finish() - } -} - - -/// An iterator over the codepoints in a `str` represented as `Utf16Char`. -#[derive(Clone)] -pub struct Utf16Chars<'a>(Utf16CharIndices<'a>); -impl<'a> From<&'a str> for Utf16Chars<'a> { - fn from(s: &str) -> Utf16Chars { - Utf16Chars(Utf16CharIndices::from(s)) - } -} -impl<'a> Utf16Chars<'a> { - /// Extract the remainder of the source `str`. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::{StrExt, Utf16Char}; - /// let mut iter = "abc".utf16chars(); - /// assert_eq!(iter.next(), Some(Utf16Char::from('a'))); - /// assert_eq!(iter.next_back(), Some(Utf16Char::from('c'))); - /// assert_eq!(iter.as_str(), "b"); - /// ``` - pub fn as_str(&self) -> &'a str { - self.0.as_str() - } -} -impl<'a> Iterator for Utf16Chars<'a> { - type Item = Utf16Char; - fn next(&mut self) -> Option<Utf16Char> { - self.0.next().map(|(_,u16c)| u16c ) - } - fn size_hint(&self) -> (usize,Option<usize>) { - self.0.size_hint() - } -} -impl<'a> DoubleEndedIterator for Utf16Chars<'a> { - fn next_back(&mut self) -> Option<Utf16Char> { - self.0.next_back().map(|(_,u16c)| u16c ) - } -} -impl<'a> fmt::Debug for Utf16Chars<'a> { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - fmtr.debug_tuple("Utf16Chars") - .field(&self.as_str()) - .finish() - } -} diff --git a/vendor/encode_unicode/src/utf8_char.rs b/vendor/encode_unicode/src/utf8_char.rs deleted file mode 100644 index 9dba4ba..0000000 --- a/vendor/encode_unicode/src/utf8_char.rs +++ /dev/null @@ -1,647 +0,0 @@ -/* Copyright 2016 The encode_unicode Developers - * - * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or - * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or - * http://opensource.org/licenses/MIT>, at your option. This file may not be - * copied, modified, or distributed except according to those terms. - */ - -use errors::{FromStrError, EmptyStrError, NonAsciiError, InvalidUtf8Slice, InvalidUtf8Array}; -use utf8_iterators::Utf8Iterator; -use traits::{CharExt, U8UtfExt}; -use utf16_char::Utf16Char; -extern crate core; -use self::core::{hash, fmt, str, ptr}; -use self::core::cmp::Ordering; -use self::core::borrow::Borrow; -use self::core::ops::Deref; -use self::core::mem::transmute; -#[cfg(feature="std")] -use self::core::iter::FromIterator; -#[cfg(feature="std")] -#[allow(deprecated)] -use std::ascii::AsciiExt; -#[cfg(feature="ascii")] -extern crate ascii; -#[cfg(feature="ascii")] -use self::ascii::{AsciiChar,ToAsciiChar,ToAsciiCharError}; - - -// I don't think there is any good default value for char, but char does. -#[derive(Default)] -// char doesn't do anything more advanced than u32 for Eq/Ord, so we shouldn't either. -// The default impl of Ord for arrays works out because longer codepoints -// start with more ones, so if they're equal, the length is the same, -// breaks down for values above 0x1f_ff_ff but those can only be created by unsafe code. -#[derive(PartialEq,Eq, PartialOrd,Ord)] - -#[derive(Clone,Copy)] - - -/// An unicode codepoint stored as UTF-8. -/// -/// It can be borrowed as a `str`, and has the same size as `char`. -pub struct Utf8Char { - bytes: [u8; 4], -} - - - ///////////////////// - //conversion traits// -///////////////////// -impl str::FromStr for Utf8Char { - type Err = FromStrError; - /// Create an `Utf8Char` from a string slice. - /// The string must contain exactly one codepoint. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::error::FromStrError::*; - /// use encode_unicode::Utf8Char; - /// use std::str::FromStr; - /// - /// assert_eq!(Utf8Char::from_str("a"), Ok(Utf8Char::from('a'))); - /// assert_eq!(Utf8Char::from_str("🂠"), Ok(Utf8Char::from('🂠'))); - /// assert_eq!(Utf8Char::from_str(""), Err(Empty)); - /// assert_eq!(Utf8Char::from_str("ab"), Err(MultipleCodepoints)); - /// assert_eq!(Utf8Char::from_str("é"), Err(MultipleCodepoints));// 'e'+u301 combining mark - /// ``` - fn from_str(s: &str) -> Result<Self, FromStrError> { - if s.is_empty() { - Err(FromStrError::Empty) - } else if s.len() != 1+s.as_bytes()[0].extra_utf8_bytes_unchecked() { - Err(FromStrError::MultipleCodepoints) - } else { - let mut bytes = [0; 4]; - bytes[..s.len()].copy_from_slice(s.as_bytes()); - Ok(Utf8Char{bytes: bytes}) - } - } -} -impl From<Utf16Char> for Utf8Char { - fn from(utf16: Utf16Char) -> Utf8Char { - match utf16.to_tuple() { - (a @ 0...0x00_7f, _) => { - Utf8Char{ bytes: [a as u8, 0, 0, 0] } - }, - (u @ 0...0x07_ff, _) => { - let b = 0x80 | (u & 0x00_3f) as u8; - let a = 0xc0 | ((u & 0x07_c0) >> 6) as u8; - Utf8Char{ bytes: [a, b, 0, 0] } - }, - (u, None) => { - let c = 0x80 | (u & 0x00_3f) as u8; - let b = 0x80 | ((u & 0x0f_c0) >> 6) as u8; - let a = 0xe0 | ((u & 0xf0_00) >> 12) as u8; - Utf8Char{ bytes: [a, b, c, 0] } - }, - (f, Some(s)) => { - let f = f + (0x01_00_00u32 >> 10) as u16; - let d = 0x80 | (s & 0x00_3f) as u8; - let c = 0x80 | ((s & 0x03_c0) >> 6) as u8 - | ((f & 0x00_03) << 4) as u8; - let b = 0x80 | ((f & 0x00_fc) >> 2) as u8; - let a = 0xf0 | ((f & 0x07_00) >> 8) as u8; - Utf8Char{ bytes: [a, b, c, d] } - } - } - } -} -impl From<char> for Utf8Char { - fn from(c: char) -> Self { - Utf8Char{ bytes: c.to_utf8_array().0 } - } -} -impl From<Utf8Char> for char { - fn from(uc: Utf8Char) -> char { - unsafe{ char::from_utf8_exact_slice_unchecked(&uc.bytes[..uc.len()]) } - } -} -impl IntoIterator for Utf8Char { - type Item=u8; - type IntoIter=Utf8Iterator; - /// Iterate over the byte values. - fn into_iter(self) -> Utf8Iterator { - Utf8Iterator::from(self) - } -} - -#[cfg(feature="std")] -impl Extend<Utf8Char> for Vec<u8> { - fn extend<I:IntoIterator<Item=Utf8Char>>(&mut self, iter: I) { - let iter = iter.into_iter(); - self.reserve(iter.size_hint().0); - for u8c in iter { - // twice as fast as self.extend_from_slice(u8c.as_bytes()); - self.push(u8c.bytes[0]); - for &extra in &u8c.bytes[1..] { - if extra != 0 { - self.push(extra); - } - } - } - } -} -#[cfg(feature="std")] -impl<'a> Extend<&'a Utf8Char> for Vec<u8> { - fn extend<I:IntoIterator<Item=&'a Utf8Char>>(&mut self, iter: I) { - self.extend(iter.into_iter().cloned()) - } -} -#[cfg(feature="std")] -impl Extend<Utf8Char> for String { - fn extend<I:IntoIterator<Item=Utf8Char>>(&mut self, iter: I) { - unsafe { self.as_mut_vec().extend(iter) } - } -} -#[cfg(feature="std")] -impl<'a> Extend<&'a Utf8Char> for String { - fn extend<I:IntoIterator<Item=&'a Utf8Char>>(&mut self, iter: I) { - self.extend(iter.into_iter().cloned()) - } -} -#[cfg(feature="std")] -impl FromIterator<Utf8Char> for String { - fn from_iter<I:IntoIterator<Item=Utf8Char>>(iter: I) -> String { - let mut string = String::new(); - string.extend(iter); - return string; - } -} -#[cfg(feature="std")] -impl<'a> FromIterator<&'a Utf8Char> for String { - fn from_iter<I:IntoIterator<Item=&'a Utf8Char>>(iter: I) -> String { - iter.into_iter().cloned().collect() - } -} -#[cfg(feature="std")] -impl FromIterator<Utf8Char> for Vec<u8> { - fn from_iter<I:IntoIterator<Item=Utf8Char>>(iter: I) -> Self { - iter.into_iter().collect::<String>().into_bytes() - } -} -#[cfg(feature="std")] -impl<'a> FromIterator<&'a Utf8Char> for Vec<u8> { - fn from_iter<I:IntoIterator<Item=&'a Utf8Char>>(iter: I) -> Self { - iter.into_iter().cloned().collect::<String>().into_bytes() - } -} - - - ///////////////// - //getter traits// -///////////////// -impl AsRef<[u8]> for Utf8Char { - fn as_ref(&self) -> &[u8] { - &self.bytes[..self.len()] - } -} -impl AsRef<str> for Utf8Char { - fn as_ref(&self) -> &str { - unsafe{ str::from_utf8_unchecked( self.as_ref() ) } - } -} -impl Borrow<[u8]> for Utf8Char { - fn borrow(&self) -> &[u8] { - self.as_ref() - } -} -impl Borrow<str> for Utf8Char { - fn borrow(&self) -> &str { - self.as_ref() - } -} -impl Deref for Utf8Char { - type Target = str; - fn deref(&self) -> &Self::Target { - self.as_ref() - } -} - - - //////////////// - //ascii traits// -//////////////// -#[cfg(feature="std")] -#[allow(deprecated)] -impl AsciiExt for Utf8Char { - type Owned = Utf8Char; - fn is_ascii(&self) -> bool { - self.bytes[0].is_ascii() - } - fn eq_ignore_ascii_case(&self, other: &Self) -> bool { - if self.is_ascii() {self.bytes[0].eq_ignore_ascii_case(&other.bytes[0])} - else {self == other} - } - fn to_ascii_uppercase(&self) -> Self::Owned { - let mut uc = *self; - uc.make_ascii_uppercase(); - uc - } - fn to_ascii_lowercase(&self) -> Self::Owned { - let mut uc = *self; - uc.make_ascii_lowercase(); - uc - } - fn make_ascii_uppercase(&mut self) { - self.bytes[0].make_ascii_uppercase() - } - fn make_ascii_lowercase(&mut self) { - self.bytes[0].make_ascii_lowercase(); - } -} - -#[cfg(feature="ascii")] -/// Requires the feature "ascii". -impl From<AsciiChar> for Utf8Char { - fn from(ac: AsciiChar) -> Self { - Utf8Char{ bytes: [ac.as_byte(),0,0,0] } - } -} -#[cfg(feature="ascii")] -/// Requires the feature "ascii". -impl ToAsciiChar for Utf8Char { - fn to_ascii_char(self) -> Result<AsciiChar, ToAsciiCharError> { - self.bytes[0].to_ascii_char() - } - unsafe fn to_ascii_char_unchecked(self) -> AsciiChar { - self.bytes[0].to_ascii_char_unchecked() - } -} - - - ///////////////////////////////////////////////////////// - //Genaral traits that cannot be derived to emulate char// -///////////////////////////////////////////////////////// -impl hash::Hash for Utf8Char { - fn hash<H : hash::Hasher>(&self, state: &mut H) { - self.to_char().hash(state); - } -} -impl fmt::Debug for Utf8Char { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self.to_char(), fmtr) - } -} -impl fmt::Display for Utf8Char { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - fmtr.write_str(self.as_str()) - } -} - - - //////////////////////////////// - //Comparisons with other types// -//////////////////////////////// -impl PartialEq<char> for Utf8Char { - fn eq(&self, u32c: &char) -> bool { - *self == Utf8Char::from(*u32c) - } -} -impl PartialEq<Utf8Char> for char { - fn eq(&self, u8c: &Utf8Char) -> bool { - Utf8Char::from(*self) == *u8c - } -} -impl PartialOrd<char> for Utf8Char { - fn partial_cmp(&self, u32c: &char) -> Option<Ordering> { - self.partial_cmp(&Self::from(*u32c)) - } -} -impl PartialOrd<Utf8Char> for char { - fn partial_cmp(&self, u8c: &Utf8Char) -> Option<Ordering> { - Utf8Char::from(*self).partial_cmp(u8c) - } -} - -impl PartialEq<Utf16Char> for Utf8Char { - fn eq(&self, u16c: &Utf16Char) -> bool { - *self == Self::from(*u16c) - } -} -impl PartialOrd<Utf16Char> for Utf8Char { - fn partial_cmp(&self, u16c: &Utf16Char) -> Option<Ordering> { - self.partial_cmp(&Self::from(*u16c)) - } -} -// The other direction is implemented in utf16_char.rs - -/// Only considers the byte equal if both it and the `Utf8Char` represents ASCII characters. -/// -/// There is no impl in the opposite direction, as this should only be used to -/// compare `Utf8Char`s against constants. -/// -/// # Examples -/// -/// ``` -/// # use encode_unicode::Utf8Char; -/// assert!(Utf8Char::from('8') == b'8'); -/// assert!(Utf8Char::from_array([0xf1,0x80,0x80,0x80]).unwrap() != 0xf1); -/// assert!(Utf8Char::from('\u{ff}') != 0xff); -/// assert!(Utf8Char::from('\u{80}') != 0x80); -/// ``` -impl PartialEq<u8> for Utf8Char { - fn eq(&self, byte: &u8) -> bool { - self.bytes[0] == *byte && self.bytes[1] == 0 - } -} -#[cfg(feature = "ascii")] -/// `Utf8Char`s that are not ASCII never compare equal. -impl PartialEq<AsciiChar> for Utf8Char { - #[inline] - fn eq(&self, ascii: &AsciiChar) -> bool { - self.bytes[0] == *ascii as u8 - } -} -#[cfg(feature = "ascii")] -/// `Utf8Char`s that are not ASCII never compare equal. -impl PartialEq<Utf8Char> for AsciiChar { - #[inline] - fn eq(&self, u8c: &Utf8Char) -> bool { - u8c == self - } -} -#[cfg(feature = "ascii")] -/// `Utf8Char`s that are not ASCII always compare greater. -impl PartialOrd<AsciiChar> for Utf8Char { - #[inline] - fn partial_cmp(&self, ascii: &AsciiChar) -> Option<Ordering> { - self.bytes[0].partial_cmp(ascii) - } -} -#[cfg(feature = "ascii")] -/// `Utf8Char`s that are not ASCII always compare greater. -impl PartialOrd<Utf8Char> for AsciiChar { - #[inline] - fn partial_cmp(&self, u8c: &Utf8Char) -> Option<Ordering> { - self.partial_cmp(&u8c.bytes[0]) - } -} - - - /////////////////////////////////////////////////////// - //pub impls that should be together for nicer rustdoc// -/////////////////////////////////////////////////////// -impl Utf8Char { - /// Create an `Utf8Char` from the first codepoint in a `str`. - /// - /// Returns an error if the `str` is empty. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::Utf8Char; - /// - /// assert_eq!(Utf8Char::from_str_start("a"), Ok((Utf8Char::from('a'),1))); - /// assert_eq!(Utf8Char::from_str_start("ab"), Ok((Utf8Char::from('a'),1))); - /// assert_eq!(Utf8Char::from_str_start("🂠 "), Ok((Utf8Char::from('🂠'),4))); - /// assert_eq!(Utf8Char::from_str_start("é"), Ok((Utf8Char::from('e'),1)));// 'e'+u301 combining mark - /// assert!(Utf8Char::from_str_start("").is_err()); - /// ``` - pub fn from_str_start(src: &str) -> Result<(Self,usize),EmptyStrError> { - unsafe { - if src.is_empty() { - Err(EmptyStrError) - } else { - Ok(Utf8Char::from_slice_start_unchecked(src.as_bytes())) - } - } - } - /// Create an `Utf8Char` of the first codepoint in an UTF-8 slice. - /// Also returns the length of the UTF-8 sequence for the codepoint. - /// - /// If the slice is from a `str`, use `::from_str_start()` to skip UTF-8 validation. - /// - /// # Errors - /// - /// Returns an `Err` if the slice is empty, doesn't start with a valid - /// UTF-8 sequence or is too short for the sequence. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::Utf8Char; - /// use encode_unicode::error::InvalidUtf8Slice::*; - /// use encode_unicode::error::InvalidUtf8::*; - /// - /// assert_eq!(Utf8Char::from_slice_start(&[b'A', b'B', b'C']), Ok((Utf8Char::from('A'),1))); - /// assert_eq!(Utf8Char::from_slice_start(&[0xdd, 0xbb]), Ok((Utf8Char::from('\u{77b}'),2))); - /// - /// assert_eq!(Utf8Char::from_slice_start(&[]), Err(TooShort(1))); - /// assert_eq!(Utf8Char::from_slice_start(&[0xf0, 0x99]), Err(TooShort(4))); - /// assert_eq!(Utf8Char::from_slice_start(&[0xee, b'F', 0x80]), Err(Utf8(NotAContinuationByte(1)))); - /// assert_eq!(Utf8Char::from_slice_start(&[0xee, 0x99, 0x0f]), Err(Utf8(NotAContinuationByte(2)))); - /// ``` - pub fn from_slice_start(src: &[u8]) -> Result<(Self,usize),InvalidUtf8Slice> { - char::from_utf8_slice_start(src).map(|(_,len)| { - let mut bytes = [0; 4]; - bytes[..len].copy_from_slice(&src[..len]); - (Utf8Char{ bytes: bytes }, len) - }) - } - /// A `from_slice_start()` that doesn't validate the codepoint. - /// - /// # Safety - /// - /// The slice must be non-empty and start with a valid UTF-8 codepoint. - /// Invalid or incomplete values might cause reads of uninitalized memory. - pub unsafe fn from_slice_start_unchecked(src: &[u8]) -> (Self,usize) { - let len = 1+src.get_unchecked(0).extra_utf8_bytes_unchecked(); - let mut bytes = [0; 4]; - ptr::copy_nonoverlapping(src.as_ptr(), &mut bytes[0] as *mut u8, len); - (Utf8Char{ bytes: bytes }, len) - } - /// Create an `Utf8Char` from a byte array after validating it. - /// - /// The codepoint must start at the first byte. - /// Unused bytes are set to zero by this function and so can be anything. - /// - /// # Errors - /// - /// Returns an `Err` if the array doesn't start with a valid UTF-8 sequence. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::Utf8Char; - /// use encode_unicode::error::InvalidUtf8Array::*; - /// use encode_unicode::error::InvalidUtf8::*; - /// use encode_unicode::error::InvalidCodepoint::*; - /// - /// assert_eq!(Utf8Char::from_array([b'A', 0, 0, 0]), Ok(Utf8Char::from('A'))); - /// assert_eq!(Utf8Char::from_array([0xf4, 0x8b, 0xbb, 0xbb]), Ok(Utf8Char::from('\u{10befb}'))); - /// assert_eq!(Utf8Char::from_array([b'A', b'B', b'C', b'D']), Ok(Utf8Char::from('A'))); - /// assert_eq!(Utf8Char::from_array([0, 0, 0xcc, 0xbb]), Ok(Utf8Char::from('\0'))); - /// - /// assert_eq!(Utf8Char::from_array([0xef, b'F', 0x80, 0x80]), Err(Utf8(NotAContinuationByte(1)))); - /// assert_eq!(Utf8Char::from_array([0xc1, 0x80, 0, 0]), Err(Utf8(OverLong))); - /// assert_eq!(Utf8Char::from_array([0xf7, 0xaa, 0x99, 0x88]), Err(Codepoint(TooHigh))); - /// ``` - pub fn from_array(utf8: [u8;4]) -> Result<Self,InvalidUtf8Array> { - unsafe { - // perform all validation - try!(char::from_utf8_array(utf8)); - let extra = utf8[0].extra_utf8_bytes_unchecked() as u32; - // zero unused bytes in one operation by transmuting the arrary to - // u32, apply an endian-corrected mask and transmute back - let mask = u32::from_le(0xff_ff_ff_ff >> 8*(3-extra)); - let unused_zeroed = mask & transmute::<_,u32>(utf8); - Ok(Utf8Char{ bytes: transmute(unused_zeroed) }) - } - } - /// Zero-cost constructor. - /// - /// # Safety - /// - /// Must contain a valid codepoint starting at the first byte, with the - /// unused bytes zeroed. - /// Bad values can easily lead to undefined behavior. - #[inline] - pub unsafe fn from_array_unchecked(utf8: [u8;4]) -> Self { - Utf8Char{ bytes: utf8 } - } - /// Create an `Utf8Char` from a single byte. - /// - /// The byte must be an ASCII character. - /// - /// # Errors - /// - /// Returns `NonAsciiError` if the byte greater than 127. - /// - /// # Examples - /// - /// ``` - /// # use encode_unicode::Utf8Char; - /// assert_eq!(Utf8Char::from_ascii(b'a').unwrap(), 'a'); - /// assert!(Utf8Char::from_ascii(128).is_err()); - /// ``` - pub fn from_ascii(ascii: u8) -> Result<Self,NonAsciiError> { - if ascii as i8 >= 0 { - Ok(Utf8Char{ bytes: [ascii, 0, 0, 0] }) - } else { - Err(NonAsciiError) - } - } - /// Create an `Utf8Char` from a single byte without checking that it's a - /// valid codepoint on its own, which is only true for ASCII characters. - /// - /// # Safety - /// - /// The byte must be less than 128. - #[inline] - pub unsafe fn from_ascii_unchecked(ascii: u8) -> Self { - Utf8Char{ bytes: [ascii, 0, 0, 0] } - } - - /// The number of bytes this character needs. - /// - /// Is between 1 and 4 (inclusive) and identical to `.as_ref().len()` or - /// `.as_char().len_utf8()`. - #[inline] - pub fn len(self) -> usize { - // Invariants of the extra bytes enambles algorithms that - // `u8.extra_utf8_bytes_unchecked()` cannot use. - // Some of them turned out to require fewer x86 instructions: - - // Exploits that unused bytes are zero and calculates the number of - // trailing zero bytes. - // Setting a bit in the first byte prevents the function from returning - // 0 for '\0' (which has 32 leading zeros). - // trailing and leading is swapped below to optimize for little-endian - // architectures. - (4 - (u32::to_le(unsafe{transmute(self.bytes)})|1).leading_zeros()/8) as usize - - // Exploits that the extra bytes have their most significant bit set if - // in use. - // Takes fewer instructions than the one above if popcnt can be used, - // (which it cannot by default, - // set RUSTFLAGS='-C target-cpu=native' to enable) - //let all: u32 = unsafe{transmute(self.bytes)}; - //let msb_mask = u32::from_be(0x00808080); - //let add_one = u32::from_be(0x80000000); - //((all & msb_mask) | add_one).count_ones() as usize - } - // There is no .is_emty() because this type is never empty. - - /// Checks that the codepoint is an ASCII character. - pub fn is_ascii(&self) -> bool { - self.bytes[0] <= 127 - } - /// Checks that two characters are an ASCII case-insensitive match. - /// - /// Is equivalent to `a.to_ascii_lowercase() == b.to_ascii_lowercase()`. - #[cfg(feature="std")] - pub fn eq_ignore_ascii_case(&self, other: &Self) -> bool { - if self.is_ascii() {self.bytes[0].eq_ignore_ascii_case(&other.bytes[0])} - else {self == other} - } - /// Converts the character to its ASCII upper case equivalent. - /// - /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', - /// but non-ASCII letters are unchanged. - #[cfg(feature="std")] - pub fn to_ascii_uppercase(&self) -> Self { - let mut uc = *self; - uc.make_ascii_uppercase(); - uc - } - /// Converts the character to its ASCII lower case equivalent. - /// - /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', - /// but non-ASCII letters are unchanged. - #[cfg(feature="std")] - pub fn to_ascii_lowercase(&self) -> Self { - let mut uc = *self; - uc.make_ascii_lowercase(); - uc - } - /// Converts the character to its ASCII upper case equivalent in-place. - /// - /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', - /// but non-ASCII letters are unchanged. - #[inline] - #[cfg(feature="std")] - pub fn make_ascii_uppercase(&mut self) { - self.bytes[0].make_ascii_uppercase() - } - /// Converts the character to its ASCII lower case equivalent in-place. - /// - /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', - /// but non-ASCII letters are unchanged. - #[inline] - #[cfg(feature="std")] - pub fn make_ascii_lowercase(&mut self) { - self.bytes[0].make_ascii_lowercase(); - } - - /// Convert from UTF-8 to UTF-32 - pub fn to_char(self) -> char { - self.into() - } - /// Write the internal representation to a slice, - /// and then returns the number of bytes written. - /// - /// # Panics - /// - /// Will panic the buffer is too small; - /// You can get the required length from `.len()`, - /// but a buffer of length four is always large enough. - pub fn to_slice(self, dst: &mut[u8]) -> usize { - if self.len() > dst.len() { - panic!("The provided buffer is too small."); - } - dst[..self.len()].copy_from_slice(&self.bytes[..self.len()]); - self.len() - } - /// Expose the internal array and the number of used bytes. - pub fn to_array(self) -> ([u8;4],usize) { - (self.bytes, self.len()) - } - /// Return a `str` view of the array the codepoint is stored as. - /// - /// Is an unambiguous version of `.as_ref()`. - pub fn as_str(&self) -> &str { - self.deref() - } -} diff --git a/vendor/encode_unicode/src/utf8_iterators.rs b/vendor/encode_unicode/src/utf8_iterators.rs deleted file mode 100644 index 891d729..0000000 --- a/vendor/encode_unicode/src/utf8_iterators.rs +++ /dev/null @@ -1,352 +0,0 @@ -/* Copyright 2016 The encode_unicode Developers - * - * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or - * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or - * http://opensource.org/licenses/MIT>, at your option. This file may not be - * copied, modified, or distributed except according to those terms. - */ - -use utf8_char::Utf8Char; -use errors::EmptyStrError; -extern crate core; -use self::core::{mem, u32, u64}; -use self::core::ops::Not; -use self::core::fmt; -use self::core::borrow::Borrow; -#[cfg(feature="std")] -use std::io::{Read, Error as ioError}; - - - -/// Read or iterate over the bytes of the UTF-8 representation of a codepoint. -#[derive(Clone)] -pub struct Utf8Iterator (u32); - -impl From<Utf8Char> for Utf8Iterator { - fn from(uc: Utf8Char) -> Self { - let used = u32::from_le(unsafe{ mem::transmute(uc.to_array().0) }); - // uses u64 because shifting an u32 by 32 bits is a no-op. - let unused_set = (u64::MAX << uc.len() as u64*8) as u32; - Utf8Iterator(used | unused_set) - } -} -impl From<char> for Utf8Iterator { - fn from(c: char) -> Self { - Self::from(Utf8Char::from(c)) - } -} -impl Iterator for Utf8Iterator { - type Item=u8; - fn next(&mut self) -> Option<u8> { - let next = self.0 as u8; - if next == 0xff { - None - } else { - self.0 = (self.0 >> 8) | 0xff_00_00_00; - Some(next) - } - } - fn size_hint(&self) -> (usize, Option<usize>) { - (self.len(), Some(self.len())) - } -} -impl ExactSizeIterator for Utf8Iterator { - fn len(&self) -> usize {// not straightforward, but possible - let unused_bytes = self.0.not().leading_zeros() / 8; - 4 - unused_bytes as usize - } -} -#[cfg(feature="std")] -impl Read for Utf8Iterator { - /// Always returns Ok - fn read(&mut self, buf: &mut[u8]) -> Result<usize, ioError> { - // Cannot call self.next() until I know I can write the result. - for (i, dst) in buf.iter_mut().enumerate() { - match self.next() { - Some(b) => *dst = b, - None => return Ok(i), - } - } - Ok(buf.len()) - } -} -impl fmt::Debug for Utf8Iterator { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - let mut content = [0; 4]; - let mut i = 0; - for b in self.clone() { - content[i] = b; - i += 1; - } - write!(fmtr, "{:?}", &content[..i]) - } -} - - - -/// Converts an iterator of `Utf8Char` (or `&Utf8Char`) -/// to an iterator of `u8`s. -/// Is equivalent to calling `.flat_map()` on the original iterator, -/// but the returned iterator is ~40% faster. -/// -/// The iterator also implements `Read` (if the `std` feature isn't disabled). -/// Reading will never produce an error, and calls to `.read()` and `.next()` -/// can be mixed. -/// -/// The exact number of bytes cannot be known in advance, but `size_hint()` -/// gives the possible range. -/// (min: all remaining characters are ASCII, max: all require four bytes) -/// -/// # Examples -/// -/// From iterator of values: -/// -/// ``` -/// use encode_unicode::{iter_bytes, CharExt}; -/// -/// let iterator = "foo".chars().map(|c| c.to_utf8() ); -/// let mut bytes = [0; 4]; -/// for (u,dst) in iter_bytes(iterator).zip(&mut bytes) {*dst=u;} -/// assert_eq!(&bytes, b"foo\0"); -/// ``` -/// -/// From iterator of references: -/// -#[cfg_attr(feature="std", doc=" ```")] -#[cfg_attr(not(feature="std"), doc=" ```no_compile")] -/// use encode_unicode::{iter_bytes, CharExt, Utf8Char}; -/// -/// let chars: Vec<Utf8Char> = "💣 bomb 💣".chars().map(|c| c.to_utf8() ).collect(); -/// let bytes: Vec<u8> = iter_bytes(&chars).collect(); -/// let flat_map: Vec<u8> = chars.iter().flat_map(|u8c| *u8c ).collect(); -/// assert_eq!(bytes, flat_map); -/// ``` -/// -/// `Read`ing from it: -/// -#[cfg_attr(feature="std", doc=" ```")] -#[cfg_attr(not(feature="std"), doc=" ```no_compile")] -/// use encode_unicode::{iter_bytes, CharExt}; -/// use std::io::Read; -/// -/// let s = "Ååh‽"; -/// assert_eq!(s.len(), 8); -/// let mut buf = [b'E'; 9]; -/// let mut reader = iter_bytes(s.chars().map(|c| c.to_utf8() )); -/// assert_eq!(reader.read(&mut buf[..]).unwrap(), 8); -/// assert_eq!(reader.read(&mut buf[..]).unwrap(), 0); -/// assert_eq!(&buf[..8], s.as_bytes()); -/// assert_eq!(buf[8], b'E'); -/// ``` -pub fn iter_bytes<U:Borrow<Utf8Char>, I:IntoIterator<Item=U>> -(iterable: I) -> Utf8CharSplitter<U, I::IntoIter> { - Utf8CharSplitter{ inner: iterable.into_iter(), prev: 0 } -} - -/// The iterator type returned by `iter_bytes()` -/// -/// See its documentation for details. -#[derive(Clone)] -pub struct Utf8CharSplitter<U:Borrow<Utf8Char>, I:Iterator<Item=U>> { - inner: I, - prev: u32, -} -impl<I:Iterator<Item=Utf8Char>> From<I> for Utf8CharSplitter<Utf8Char,I> { - /// A less generic constructor than `iter_bytes()` - fn from(iter: I) -> Self { - iter_bytes(iter) - } -} -impl<U:Borrow<Utf8Char>, I:Iterator<Item=U>> Utf8CharSplitter<U,I> { - /// Extracts the source iterator. - /// - /// Note that `iter_bytes(iter.into_inner())` is not a no-op: - /// If the last returned byte from `next()` was not an ASCII by, - /// the remaining bytes of that codepoint is lost. - pub fn into_inner(self) -> I { - self.inner - } -} -impl<U:Borrow<Utf8Char>, I:Iterator<Item=U>> Iterator for Utf8CharSplitter<U,I> { - type Item = u8; - fn next(&mut self) -> Option<Self::Item> { - if self.prev == 0 { - self.inner.next().map(|u8c| { - let array = u8c.borrow().to_array().0; - self.prev = unsafe{ u32::from_le(mem::transmute(array)) } >> 8; - array[0] - }) - } else { - let next = self.prev as u8; - self.prev >>= 8; - Some(next) - } - } - fn size_hint(&self) -> (usize,Option<usize>) { - // Doesn't need to handle unlikely overflows correctly because - // size_hint() cannot be relied upon anyway. (the trait isn't unsafe) - let (min, max) = self.inner.size_hint(); - let add = 4 - (self.prev.leading_zeros() / 8) as usize; - (min.wrapping_add(add), max.map(|max| max.wrapping_mul(4).wrapping_add(add) )) - } -} -#[cfg(feature="std")] -impl<U:Borrow<Utf8Char>, I:Iterator<Item=U>> Read for Utf8CharSplitter<U,I> { - /// Always returns `Ok` - fn read(&mut self, buf: &mut[u8]) -> Result<usize, ioError> { - let mut i = 0; - // write remaining bytes of previous codepoint - while self.prev != 0 && i < buf.len() { - buf[i] = self.prev as u8; - self.prev >>= 8; - i += 1; - } - // write whole characters - while i < buf.len() { - let bytes = match self.inner.next() { - Some(u8c) => u8c.borrow().to_array().0, - None => break - }; - buf[i] = bytes[0]; - i += 1; - if bytes[1] != 0 { - let len = bytes[0].not().leading_zeros() as usize; - let mut written = 1; - while written < len { - if i < buf.len() { - buf[i] = bytes[written]; - i += 1; - written += 1; - } else { - let bytes_as_u32 = unsafe{ u32::from_le(mem::transmute(bytes)) }; - self.prev = bytes_as_u32 >> (8*written); - return Ok(i); - } - } - } - } - Ok(i) - } -} - - - -/// An iterator over the `Utf8Char` of a string slice, and their positions. -/// -/// This struct is created by the `utf8char_indices() method from [`StrExt`] trait. See its documentation for more. -#[derive(Clone)] -pub struct Utf8CharIndices<'a>{ - str: &'a str, - index: usize, -} -impl<'a> From<&'a str> for Utf8CharIndices<'a> { - fn from(s: &str) -> Utf8CharIndices { - Utf8CharIndices{str: s, index: 0} - } -} -impl<'a> Utf8CharIndices<'a> { - /// Extract the remainder of the source `str`. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::{StrExt, Utf8Char}; - /// let mut iter = "abc".utf8char_indices(); - /// assert_eq!(iter.next_back(), Some((2, Utf8Char::from('c')))); - /// assert_eq!(iter.next(), Some((0, Utf8Char::from('a')))); - /// assert_eq!(iter.as_str(), "b"); - /// ``` - pub fn as_str(&self) -> &'a str { - &self.str[self.index..] - } -} -impl<'a> Iterator for Utf8CharIndices<'a> { - type Item = (usize,Utf8Char); - fn next(&mut self) -> Option<(usize,Utf8Char)> { - match Utf8Char::from_str_start(&self.str[self.index..]) { - Ok((u8c, len)) => { - let item = (self.index, u8c); - self.index += len; - Some(item) - }, - Err(EmptyStrError) => None - } - } - fn size_hint(&self) -> (usize,Option<usize>) { - let len = self.str.len() - self.index; - // For len+3 to overflow, the slice must fill all but two bytes of - // addressable memory, and size_hint() doesn't need to be correct. - (len.wrapping_add(3)/4, Some(len)) - } -} -impl<'a> DoubleEndedIterator for Utf8CharIndices<'a> { - fn next_back(&mut self) -> Option<(usize,Utf8Char)> { - // Cannot refactor out the unwrap without switching to ::from_slice() - // since slicing the str panics if not on a boundary. - if self.index < self.str.len() { - let rev = self.str.bytes().rev(); - let len = 1 + rev.take_while(|b| b & 0b1100_0000 == 0b1000_0000 ).count(); - let starts = self.str.len() - len; - let (u8c,_) = Utf8Char::from_str_start(&self.str[starts..]).unwrap(); - self.str = &self.str[..starts]; - Some((starts, u8c)) - } else { - None - } - } -} -impl<'a> fmt::Debug for Utf8CharIndices<'a> { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - fmtr.debug_tuple("Utf8CharIndices") - .field(&self.index) - .field(&self.as_str()) - .finish() - } -} - - -/// An iterator over the codepoints in a `str` represented as `Utf8Char`. -#[derive(Clone)] -pub struct Utf8Chars<'a>(Utf8CharIndices<'a>); -impl<'a> From<&'a str> for Utf8Chars<'a> { - fn from(s: &str) -> Utf8Chars { - Utf8Chars(Utf8CharIndices::from(s)) - } -} -impl<'a> Utf8Chars<'a> { - /// Extract the remainder of the source `str`. - /// - /// # Examples - /// - /// ``` - /// use encode_unicode::{StrExt, Utf8Char}; - /// let mut iter = "abc".utf8chars(); - /// assert_eq!(iter.next(), Some(Utf8Char::from('a'))); - /// assert_eq!(iter.next_back(), Some(Utf8Char::from('c'))); - /// assert_eq!(iter.as_str(), "b"); - /// ``` - pub fn as_str(&self) -> &'a str { - self.0.as_str() - } -} -impl<'a> Iterator for Utf8Chars<'a> { - type Item = Utf8Char; - fn next(&mut self) -> Option<Utf8Char> { - self.0.next().map(|(_,u8c)| u8c ) - } - fn size_hint(&self) -> (usize,Option<usize>) { - self.0.size_hint() - } -} -impl<'a> DoubleEndedIterator for Utf8Chars<'a> { - fn next_back(&mut self) -> Option<Utf8Char> { - self.0.next_back().map(|(_,u8c)| u8c ) - } -} -impl<'a> fmt::Debug for Utf8Chars<'a> { - fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { - fmtr.debug_tuple("Utf8CharIndices") - .field(&self.as_str()) - .finish() - } -} diff --git a/vendor/encode_unicode/tests/errs.rs b/vendor/encode_unicode/tests/errs.rs deleted file mode 100644 index 9c8747b..0000000 --- a/vendor/encode_unicode/tests/errs.rs +++ /dev/null @@ -1,197 +0,0 @@ -/* Copyright 2016 The encode_unicode Developers - * - * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or - * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or - * http://opensource.org/licenses/MIT>, at your option. This file may not be - * copied, modified, or distributed except according to those terms. - */ - -//! Test that methods gives the correct error. -//! Some also test a bit more because it's easy. - -extern crate core; -use core::char; -extern crate encode_unicode; -use encode_unicode::*; -use encode_unicode::error::*; -use encode_unicode::error::InvalidUtf8Array as a; -use encode_unicode::error::InvalidUtf8Slice as s; -use encode_unicode::error::InvalidCodepoint::*; -use encode_unicode::error::InvalidUtf8::*; -use encode_unicode::error::InvalidUtf8FirstByte::*; - - -#[test] -fn from_u32() { - for c in 0xd800..0xe000 { - assert_eq!(char::from_u32_detailed(c), Err(Utf16Reserved)); - } - let mut c = 0x11_00_00; - loop { - assert_eq!(char::from_u32_detailed(c), Err(TooHigh)); - // Don't test every value. (Range.step_by() is unstable) - match c.checked_add(0x10_11_11) { - Some(next) => c = next, - None => break, - } - } -} - -#[test] -fn utf8_extra_bytes() { - for c in 0..256 { - assert_eq!( (c as u8).extra_utf8_bytes(), match c { - 0b_1000_0000...0b_1011_1111 => Err(ContinuationByte), - 0b_1111_1000...0b_1111_1111 => Err(TooLongSeqence), - 0b_0000_0000...0b_0111_1111 => Ok(0), - 0b_1100_0000...0b_1101_1111 => Ok(1), - 0b_1110_0000...0b_1110_1111 => Ok(2), - 0b_1111_0000...0b_1111_0111 => Ok(3), - _ => unreachable!(), - }); - } -} - -#[test] -fn utf16_extra_unit() { - for c in 0..0x1_00_00 { - assert_eq!( (c as u16).utf16_needs_extra_unit(), match c { - 0b_0000_0000_0000_0000...0b_1101_0111_1111_1111 => Ok(false), - 0b_1101_1000_0000_0000...0b_1101_1011_1111_1111 => Ok(true), - 0b_1101_1100_0000_0000...0b_1101_1111_1111_1111 => Err(InvalidUtf16FirstUnit), - 0b_1110_0000_0000_0000...0b_1111_1111_1111_1111 => Ok(false), - _ => unreachable!(), - }); - } -} - - -#[test] -fn from_utf16_tuple() { - use encode_unicode::error::InvalidUtf16Tuple::*; - for u in 0xdc00..0xe000 { - let close = if u%3==0 {u-100} else {u+100}; - let doesnt_matter = if u%2==0 {Some(close)} else {None}; - assert_eq!(char::from_utf16_tuple((u,doesnt_matter)), Err(FirstIsTrailingSurrogate)); - } - for u in (0..0xd800).chain(0xe000..0x10000) { - assert_eq!( - char::from_utf16_tuple((u as u16, Some((0x100+u) as u16))), - Err(SuperfluousSecond) - ); - } - for u in 0xd800..0xdc00 { - assert_eq!(char::from_utf16_tuple((u,None)), Err(MissingSecond)); - assert_eq!(char::from_utf16_tuple((u,Some(u - 0x2ff))), Err(InvalidSecond)); - } -} - -#[test] -fn from_utf16_slice_start() { - use encode_unicode::error::InvalidUtf16Slice::*; - assert_eq!(char::from_utf16_slice_start(&[]), Err(EmptySlice)); - let mut buf = [0; 6]; - for u in 0xd800..0xdc00 { - buf[0] = u; - assert_eq!(char::from_utf16_slice_start(&buf[..1]), Err(MissingSecond)); - buf[1] = u; - let pass = 2 + (u as usize % (buf.len()-2)); - assert_eq!(char::from_utf16_slice_start(&buf[..pass]), Err(SecondNotLowSurrogate)); - } - for u in 0xdc00..0xe000 { - buf[0] = u; - let close = if u%3==0 {u-100} else {u+100}; - let pass = 1 + (u as usize % (buf.len()-1)); - buf[pass] = close; - assert_eq!(char::from_utf16_slice_start(&buf[..pass]), Err(FirstLowSurrogate)); - } -} - -#[test] -fn utf8_overlong() { - let overlongs = [ - [0xf0,0x8f], [0xf0,0x87], [0xf0,0x80], // 4-byte - [0xe0,0x9f], [0xe0,0x8f], [0xe0,0x80], // 3-byte - [0xc1,0xbf], [0xc1,0x92], [0xc1,0x80], // 2-byte - [0xc0,0xbf], [0xc0,0x9f], [0xc0,0x80], // 2-byte - ]; - for o in overlongs.iter() { - for &last in &[0x80, 0xbf] { - let arr = [o[0], o[1], last, last]; - assert_eq!(char::from_utf8_slice_start(&arr), Err(InvalidUtf8Slice::Utf8(OverLong))); - assert_eq!(char::from_utf8_array(arr), Err(InvalidUtf8Array::Utf8(OverLong))); - assert_eq!(Utf8Char::from_slice_start(&arr), Err(InvalidUtf8Slice::Utf8(OverLong))); - assert_eq!(Utf8Char::from_array(arr), Err(InvalidUtf8Array::Utf8(OverLong))); - } - } -} - -#[test] -fn from_str_start() { - assert_eq!(Utf8Char::from_str_start(""), Err(EmptyStrError)); - assert_eq!(Utf16Char::from_str_start(""), Err(EmptyStrError)); -} - -#[test] fn utf8_codepoint_is_too_high() { - assert_eq!(Utf8Char::from_array([0xf4, 0x90, 0x80, 0x80]), Err(a::Codepoint(TooHigh))); - assert_eq!(char::from_utf8_array([0xf4, 0x90, 0x80, 0x80]), Err(a::Codepoint(TooHigh))); - assert_eq!(Utf8Char::from_slice_start(&[0xf4, 0x90, 0x80, 0x80]), Err(s::Codepoint(TooHigh))); - assert_eq!(char::from_utf8_slice_start(&[0xf4, 0x90, 0x80, 0x80]), Err(s::Codepoint(TooHigh))); - - assert_eq!(Utf8Char::from_array([0xf5, 0x88, 0x99, 0xaa]), Err(a::Codepoint(TooHigh))); - assert_eq!(char::from_utf8_array([0xf5, 0xaa, 0xbb, 0x88]), Err(a::Codepoint(TooHigh))); - assert_eq!(Utf8Char::from_slice_start(&[0xf5, 0x99, 0xaa, 0xbb]), Err(s::Codepoint(TooHigh))); - assert_eq!(char::from_utf8_slice_start(&[0xf5, 0xbb, 0x88, 0x99]), Err(s::Codepoint(TooHigh))); -} - -#[test] fn utf8_codepoint_is_utf16_reserved() { - assert_eq!(Utf8Char::from_array([0xed, 0xa0, 0x80, 0xff]), Err(a::Codepoint(Utf16Reserved))); - assert_eq!(char::from_utf8_array([0xed, 0xa0, 0x8f, 0x00]), Err(a::Codepoint(Utf16Reserved))); - assert_eq!(Utf8Char::from_slice_start(&[0xed, 0xa0, 0xbe, 0xa5]), Err(s::Codepoint(Utf16Reserved))); - assert_eq!(char::from_utf8_slice_start(&[0xed, 0xa0, 0xbf]), Err(s::Codepoint(Utf16Reserved))); - assert_eq!(Utf8Char::from_array([0xed, 0xbf, 0x80, 0xff]), Err(a::Codepoint(Utf16Reserved))); - assert_eq!(char::from_utf8_array([0xed, 0xbf, 0x8f, 0x00]), Err(a::Codepoint(Utf16Reserved))); - assert_eq!(Utf8Char::from_slice_start(&[0xed, 0xbf, 0xbe, 0xa5]), Err(s::Codepoint(Utf16Reserved))); - assert_eq!(char::from_utf8_slice_start(&[0xed, 0xbf, 0xbf]), Err(s::Codepoint(Utf16Reserved))); -} - -#[test] fn utf8_first_is_continuation_byte() { - for first in 0x80..0xc0 { - let arr = [first, first<<2, first<<4, first<<6]; - assert_eq!(Utf8Char::from_array(arr), Err(a::Utf8(FirstByte(ContinuationByte)))); - assert_eq!(char::from_utf8_array(arr), Err(a::Utf8(FirstByte(ContinuationByte)))); - let len = (1 + first%3) as usize; - assert_eq!(Utf8Char::from_slice_start(&arr[..len]), Err(s::Utf8(FirstByte(ContinuationByte)))); - assert_eq!(char::from_utf8_slice_start(&arr[..len]), Err(s::Utf8(FirstByte(ContinuationByte)))); - } -} - -#[test] fn utf8_too_long() { - for first in 0xf8..0x100 { - let arr = [first as u8, 0x88, 0x80, 0x80]; - assert_eq!(Utf8Char::from_array(arr), Err(a::Utf8(FirstByte(TooLongSeqence)))); - assert_eq!(char::from_utf8_array(arr), Err(a::Utf8(FirstByte(TooLongSeqence)))); - let arr = [first as u8, 0x88, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80]; - let slice = &arr[..if first&1 == 0 {1} else {8}]; - assert_eq!(Utf8Char::from_slice_start(slice), Err(s::Utf8(FirstByte(TooLongSeqence)))); - assert_eq!(char::from_utf8_slice_start(slice), Err(s::Utf8(FirstByte(TooLongSeqence)))); - } -} - -#[test] fn utf8_not_continuation_byte() { - for first in 0xc2..0xf4 { - let mut arr = [first, 0x90, 0xa0, 0xb0]; - let extra = first.extra_utf8_bytes().unwrap(); - for corrupt in (1..extra).rev() { - let expected = NotAContinuationByte(corrupt); - for &bad in &[0x00, 0x3f, 0x40, 0x7f, 0xc0, 0xff] { - arr[corrupt] = bad; - assert_eq!(Utf8Char::from_array(arr), Err(a::Utf8(expected)), "{:?}", arr); - assert_eq!(char::from_utf8_array(arr), Err(a::Utf8(expected))); - let slice = if first&1 == 0 {&arr[..1+extra]} else {&arr}; - assert_eq!(Utf8Char::from_slice_start(slice), Err(s::Utf8(expected)), "{:?}", slice); - assert_eq!(char::from_utf8_slice_start(slice), Err(s::Utf8(expected))); - } - } - } -} diff --git a/vendor/encode_unicode/tests/exhaustive.rs b/vendor/encode_unicode/tests/exhaustive.rs deleted file mode 100644 index ce60c6b..0000000 --- a/vendor/encode_unicode/tests/exhaustive.rs +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright 2018 The encode_unicode Developers - * - * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or - * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or - * http://opensource.org/licenses/MIT>, at your option. This file may not be - * copied, modified, or distributed except according to those terms. - */ - -//! Tests that try all possible values for at least one parameter / byte / unit -//! of the tested function. - -use std::char; -extern crate encode_unicode; -use encode_unicode::*; - -#[test] -fn from_ascii() { - for cp in 0u32..256 { - assert_eq!(Utf8Char::from_ascii(cp as u8).is_ok(), cp & 0x80 == 0); - if let Ok(u8c) = Utf8Char::from_ascii(cp as u8) { - assert_eq!(u8c, Utf8Char::from(cp as u8 as char)); - } - } -} - -#[test] -fn from_bmp() { - for cp in 0u32..0x1_00_00 { - assert_eq!( - Utf16Char::from_bmp(cp as u16).ok(), - char::from_u32(cp).map(|u32c| Utf16Char::from(u32c) ) - ); - } -} diff --git a/vendor/encode_unicode/tests/iterators.rs b/vendor/encode_unicode/tests/iterators.rs deleted file mode 100644 index 2a4c006..0000000 --- a/vendor/encode_unicode/tests/iterators.rs +++ /dev/null @@ -1,182 +0,0 @@ -/* Copyright 2018 The encode_unicode Developers - * - * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or - * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or - * http://opensource.org/licenses/MIT>, at your option. This file may not be - * copied, modified, or distributed except according to those terms. - */ - -//! Iterator tests - -#![cfg(feature="std")] - -extern crate encode_unicode; - -use encode_unicode::{IterExt, SliceExt, CharExt}; -use encode_unicode::iterator::Utf8CharSplitter; -use encode_unicode::error::InvalidUtf8Slice::*; -use encode_unicode::error::InvalidUtf8::*; -use encode_unicode::error::InvalidUtf8FirstByte::*; -use encode_unicode::error::InvalidCodepoint::*; -use encode_unicode::error::Utf16PairError::*; -use std::io::Read; -use std::cmp::min; - -#[test] fn utf8charmerger() { - let slice = b"\xf0\xa1\x92X\xcc\xbb"; - let mut iter = slice.iter().to_utf8chars(); - assert_eq!(iter.size_hint(), (1, Some(6))); - assert_eq!(format!("{:?}", &iter), - format!("Utf8CharMerger {{ buffered: [], inner: {:?} }}", slice.iter())); - - assert_eq!(iter.next(), Some(Err(Utf8(NotAContinuationByte(3))))); - assert_eq!(iter.size_hint(), (0, Some(5))); - assert_eq!( - format!("{:?}", &iter), - format!("Utf8CharMerger {{ buffered: [161, 146, 88], inner: {:?} }}", slice[4..].iter()) - ); - - assert_eq!(iter.next(), Some(Err(Utf8(FirstByte(ContinuationByte))))); - assert_eq!(iter.into_inner().next(), Some(&b'\xcc')); -} - -#[test] fn utf8chardecoder() { - let slice = b"\xf4\xbf\x80\x80XY\xcc\xbbZ_"; - let mut iter = slice.utf8char_indices(); - assert_eq!(iter.size_hint(), (2, Some(10))); - assert_eq!( - format!("{:?}", &iter), - format!("Utf8CharDecoder {{ bytes[0..]: {:?} }}", &slice) - ); - - assert_eq!(iter.next(), Some((0, Err(Codepoint(TooHigh)), 1))); - assert_eq!( - format!("{:?}", &iter), - format!("Utf8CharDecoder {{ bytes[1..]: {:?} }}", &slice[1..]) - ); - assert_eq!(iter.size_hint(), (2, Some(9))); - assert_eq!(iter.count(), 8); -} - -#[test] fn utf16charmerger() { - let slice = [0xd800, 'x' as u16, 0xd900, 0xdfff, 'λ' as u16]; - let mut iter = slice.iter().to_utf16chars(); - assert_eq!(iter.size_hint(), (2, Some(5))); - assert_eq!(format!("{:?}", &iter), - format!("Utf16CharMerger {{ buffered: None, inner: {:?} }}", slice.iter())); - - assert_eq!(iter.next(), Some(Err(UnmatchedLeadingSurrogate))); - assert_eq!(iter.size_hint(), (1, Some(4))); - assert_eq!( - format!("{:?}", &iter), - format!("Utf16CharMerger {{ buffered: Some(120), inner: {:?} }}", slice[2..].iter()) - ); - - assert_eq!(iter.into_inner().next(), Some(&0xd900)); -} - -#[test] fn utf16chardecoder() { - let slice = [0xd800, 'x' as u16, 0xd900, 0xdfff, 'λ' as u16]; - let mut iter = slice.utf16char_indices(); - assert_eq!(iter.size_hint(), (2, Some(5))); - assert_eq!( - format!("{:?}", &iter), - format!("Utf16CharDecoder {{ units[0..]: {:?} }}", &slice) - ); - - assert_eq!(iter.next(), Some((0, Err(UnmatchedLeadingSurrogate), 1))); - assert_eq!( - format!("{:?}", &iter), - format!("Utf16CharDecoder {{ units[1..]: {:?} }}", &slice[1..]) - ); - assert_eq!(iter.size_hint(), (2, Some(4))); - assert_eq!(iter.count(), 3); -} - - - -/// Tests for ensuring that iterators which also implement Read support -/// interleaving calls of `read()` and `next()`, and that they implement Read -/// correctly (support any buffer size at any time). - -#[test] fn read_single_ascii() { - let uc = 'a'.to_utf8(); - assert_eq!(uc.len(), 1); - for chunk in 1..5 { - let mut buf = [b'E'; 6]; - let mut iter = uc.into_iter(); - let mut written = 0; - for _ in 0..4 { - assert_eq!(iter.read(&mut buf[..0]).unwrap(), 0); - let wrote = iter.read(&mut buf[written..written+chunk]).unwrap(); - assert_eq!(wrote, min(1-written, chunk)); - written += wrote; - for &b in &buf[written..] {assert_eq!(b, b'E');} - assert_eq!(buf[..written], AsRef::<[u8]>::as_ref(&uc)[..written]); - } - assert_eq!(written, 1); - } -} - -#[test] fn read_single_nonascii() { - let uc = 'ä'.to_utf8(); - assert_eq!(uc.len(), 2); - for chunk in 1..5 { - let mut buf = [b'E'; 6]; - let mut iter = uc.into_iter(); - let mut written = 0; - for _ in 0..4 { - assert_eq!(iter.read(&mut buf[..0]).unwrap(), 0); - let wrote = iter.read(&mut buf[written..written+chunk]).unwrap(); - assert_eq!(wrote, min(2-written, chunk)); - written += wrote; - for &b in &buf[written..] {assert_eq!(b, b'E');} - assert_eq!(buf[..written], AsRef::<[u8]>::as_ref(&uc)[..written]); - } - assert_eq!(written, 2); - } -} - - -#[test] fn utf8charsplitter_read_all_sizes() { - let s = "1111\u{104444}\u{222}1\u{833}1111\u{100004}"; - assert!(s.len()%3 == 1); - let mut buf = vec![b'E'; s.len()+6]; - for size in 2..6 {//s.len()+4 { - let mut reader = Utf8CharSplitter::from(s.chars().map(|c| c.to_utf8() )); - for (offset, part) in s.as_bytes().chunks(size).enumerate() { - let read_to = if part.len() == size {(offset+1)*size} else {buf.len()}; - assert_eq!(reader.read(&mut buf[offset*size..read_to]).unwrap(), part.len()); - assert_eq!(&buf[..offset*size+part.len()], &s.as_bytes()[..offset*size+part.len()]); - } - assert_eq!(reader.read(&mut buf[..]).unwrap(), 0); - assert!(buf[s.len()..].iter().all(|&b| b==b'E' )); - } -} - -#[test] fn utf8charsplitter_alternate_iter_read() { - let s = "1111\u{104444}\u{222}1\u{833}1111\u{100004}"; - let mut buf = [b'0'; 10]; - for n in 0..2 { - // need to collect to test size_hint() - // because chars().size_hint() returns ((bytes+3)/4, Some(bytes)) - let u8chars = s.chars().map(|c| c.to_utf8() ).collect::<Vec<_>>(); - let mut iter: Utf8CharSplitter<_,_> = u8chars.into_iter().into(); - for (i, byte) in s.bytes().enumerate() { - let until_next = s.as_bytes()[i..].iter().take_while(|&b| (b>>6)==0b10u8 ).count(); - let remaining_chars = s[i+until_next..].chars().count(); - println!("{}. run: byte {:02} of {}, remaining: {:02}+{}: 0b{:08b} = {:?}", - n, i, s.len(), remaining_chars, until_next, byte, byte as char); - assert_eq!(iter.read(&mut[][..]).unwrap(), 0); - if i % 2 == n { - assert_eq!(iter.next(), Some(byte)); - } else { - assert_eq!(iter.read(&mut buf[..1]).unwrap(), 1); - assert_eq!(buf[0], byte); - } - } - assert_eq!(iter.size_hint(), (0, Some(0))); - assert_eq!(iter.next(), None); - assert_eq!(iter.read(&mut buf[..]).unwrap(), 0); - } -} diff --git a/vendor/encode_unicode/tests/oks.rs b/vendor/encode_unicode/tests/oks.rs deleted file mode 100644 index 8bab37f..0000000 --- a/vendor/encode_unicode/tests/oks.rs +++ /dev/null @@ -1,284 +0,0 @@ -/* Copyright 2016 The encode_unicode Developers - * - * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or - * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or - * http://opensource.org/licenses/MIT>, at your option. This file may not be - * copied, modified, or distributed except according to those terms. - */ - -//! Test that every method gives the correct result for valid values. -//! Except iterators, which are stateful. - -use std::char; -use std::str::{self,FromStr}; -use std::cmp::Ordering; -use std::hash::{Hash,Hasher}; -use std::collections::hash_map::DefaultHasher; -#[allow(deprecated,unused)] -use std::ascii::AsciiExt; -use std::iter::FromIterator; -extern crate encode_unicode; -use encode_unicode::*; - - -#[test] -fn equal_defaults() { - assert_eq!(Utf8Char::default().to_char(), char::default()); - assert_eq!(Utf16Char::default().to_char(), char::default()); -} - -#[test] -fn same_size_as_char() { - use std::mem::size_of; - assert_eq!(size_of::<Utf8Char>(), size_of::<char>()); - assert_eq!(size_of::<Utf16Char>(), size_of::<char>()); -} - -#[test] -fn utf16chars_to_string() { - let s = "aå\u{10ffff}‽\u{100000}\u{fee1}"; - let u16cs = s.chars().map(|c| Utf16Char::from(c) ).collect::<Vec<Utf16Char>>(); - - let mut from_refs: String = u16cs.iter().collect(); - assert_eq!(&from_refs, s); - from_refs.extend(&u16cs); - assert_eq!(&from_refs[s.len()..], s); - - let mut from_vals: String = u16cs.iter().cloned().collect(); - assert_eq!(&from_vals, s); - from_vals.extend(u16cs); - assert_eq!(&from_vals[s.len()..], s); -} - - -const EDGES_AND_BETWEEN: [char;19] = [ - '\u{0}',// min - '\u{3b}',// middle ASCII - 'A',// min ASCII uppercase - 'N',// middle ASCII uppercase - 'Z',// max ASCII uppercase - 'a',// min ASCII lowercase - 'm',// middle ASCII lowercase - 'z',// max ASCII lowercase - '\u{7f}',// max ASCII and 1-byte UTF-8 - '\u{80}',// min 2-byte UTF-8 - '\u{111}',// middle - '\u{7ff}',// max 2-byte UTF-8 - '\u{800}',// min 3-byte UTF-8 - '\u{d7ff}',// before reserved - '\u{e000}',// after reserved - '\u{ffff}',// max UTF-16 single and 3-byte UTF-8 - '\u{10000}',// min UTF-16 surrogate and 4-byte UTF-8 - '\u{abcde}',// middle - '\u{10ffff}',// max -]; - -fn eq_cmp_hash(c: char) -> (Utf8Char, Utf16Char) { - fn hash<T:Hash>(v: T) -> u64 { - #[allow(deprecated)] - let mut hasher = DefaultHasher::new(); - v.hash(&mut hasher); - hasher.finish() - } - let u8c = c.to_utf8(); - assert_eq!(u8c.to_char(), c); - assert_eq!(u8c, u8c); - assert_eq!(hash(u8c), hash(u8c)); - assert_eq!(u8c.cmp(&u8c), Ordering::Equal); - assert!(u8c.eq_ignore_ascii_case(&u8c)); - let u16c = c.to_utf16(); - assert_eq!(u16c.to_char(), c); - assert_eq!(u16c, u16c); - assert_eq!(hash(u16c), hash(c)); - assert_eq!(u16c.cmp(&u16c), Ordering::Equal); - assert!(u16c.eq_ignore_ascii_case(&u16c)); - - assert_eq!(u8c, c); - assert_eq!(c, u8c); - assert_eq!(u16c, c); - assert_eq!(c, u16c); - assert_eq!(u8c, u16c); - assert_eq!(u16c, u8c); - assert_eq!(u8c == c as u8, c <= '\u{7F}'); - assert_eq!(u16c == c as u8, c <= '\u{FF}'); - assert_eq!(u16c == c as u16, c <= '\u{FFFF}'); - - assert_eq!(u8c.partial_cmp(&c), Some(Ordering::Equal)); - assert_eq!(c.partial_cmp(&u8c), Some(Ordering::Equal)); - assert_eq!(u16c.partial_cmp(&c), Some(Ordering::Equal)); - assert_eq!(c.partial_cmp(&u16c), Some(Ordering::Equal)); - assert_eq!(u8c.partial_cmp(&u16c), Some(Ordering::Equal)); - assert_eq!(u16c.partial_cmp(&u8c), Some(Ordering::Equal)); - - - for &other in &EDGES_AND_BETWEEN { - let u8other = other.to_utf8(); - assert_eq!(u8c == u8other, c == other); - assert_eq!(hash(u8c)==hash(u8other), hash(c)==hash(other)); - assert_eq!(u8c.cmp(&u8other), c.cmp(&other)); - assert_eq!(u8c.eq_ignore_ascii_case(&u8other), c.eq_ignore_ascii_case(&other)); - assert_eq!(u8c.partial_cmp(&other), c.partial_cmp(&other)); - assert_eq!(c.partial_cmp(&u8other), c.partial_cmp(&other)); - assert_eq!(u8other.partial_cmp(&c), other.partial_cmp(&c)); - assert_eq!(other.partial_cmp(&u8c), other.partial_cmp(&c)); - assert_eq!(u8c == other as u8, other as u8 <= 127 && c == other as u8 as char); - - let u16other = other.to_utf16(); - assert_eq!(u16c == u16other, c == other); - assert_eq!(hash(u16c)==hash(u16other), hash(c)==hash(other)); - assert_eq!(u16c.cmp(&u16other), c.cmp(&other)); - assert_eq!(u16c.eq_ignore_ascii_case(&u16other), c.eq_ignore_ascii_case(&other)); - assert_eq!(u16c.partial_cmp(&other), c.partial_cmp(&other)); - assert_eq!(c.partial_cmp(&u16other), c.partial_cmp(&other)); - assert_eq!(u16other.partial_cmp(&c), other.partial_cmp(&c)); - assert_eq!(other.partial_cmp(&u16c), other.partial_cmp(&c)); - assert_eq!(u16c == other as u8, c == other as u8 as char); - assert_eq!(u16c == other as u16, c as u32 == other as u16 as u32); - - assert_eq!(u8c == u16other, c == other); - assert_eq!(u16c == u8other, c == other); - assert_eq!(u8c.partial_cmp(&u16other), c.partial_cmp(&other)); - assert_eq!(u16c.partial_cmp(&u8other), c.partial_cmp(&other)); - assert_eq!(u8other.partial_cmp(&u16c), other.partial_cmp(&c)); - assert_eq!(u16other.partial_cmp(&u8c), other.partial_cmp(&c)); - } - (u8c, u16c) -} - -fn iterators(c: char) { - let mut iter = c.iter_utf8_bytes(); - let mut buf = [0; 4]; - let mut iter_ref = c.encode_utf8(&mut buf[..]).as_bytes().iter(); - for _ in 0..6 { - assert_eq!(iter.size_hint(), iter_ref.size_hint()); - assert_eq!(format!("{:?}", iter), format!("{:?}", iter_ref.as_slice())); - assert_eq!(iter.next(), iter_ref.next().cloned()); - } - - let mut iter = c.iter_utf16_units(); - let mut buf = [0; 2]; - let mut iter_ref = c.encode_utf16(&mut buf[..]).iter(); - for _ in 0..4 { - assert_eq!(iter.size_hint(), iter_ref.size_hint()); - assert_eq!(format!("{:?}", iter), format!("{:?}", iter_ref.as_slice())); - assert_eq!(iter.next(), iter_ref.next().cloned()); - } -} - -fn test(c: char) { - assert_eq!(char::from_u32(c as u32), Some(c)); - assert_eq!(char::from_u32_detailed(c as u32), Ok(c)); - assert_eq!(unsafe{ char::from_u32_unchecked(c as u32) }, c); - let (u8c, u16c) = eq_cmp_hash(c); - iterators(c); - assert_eq!(Utf16Char::from(u8c), u16c); - assert_eq!(Utf8Char::from(u16c), u8c); - let utf8_len = c.len_utf8(); - let utf16_len = c.len_utf16(); - let mut as_str = c.to_string(); - - // UTF-8 - let mut buf = [0; 4]; - let reference = c.encode_utf8(&mut buf[..]).as_bytes(); - let len = reference.len(); // short name because it is used in many places. - assert_eq!(len, utf8_len); - assert_eq!(reference[0].extra_utf8_bytes(), Ok(len-1)); - assert_eq!(reference[0].extra_utf8_bytes_unchecked(), len-1); - assert_eq!(AsRef::<[u8]>::as_ref(&u8c), reference); - - let (arr,arrlen) = u8c.to_array(); - assert_eq!(arrlen, len); - assert_eq!(Utf8Char::from_array(arr), Ok(u8c)); - assert_eq!(c.to_utf8_array(), (arr, len)); - - let str_ = str::from_utf8(reference).unwrap(); - let ustr = Utf8Char::from_str(str_).unwrap(); - assert_eq!(ustr.to_array().0, arr);// bitwise equality - assert_eq!(char::from_utf8_array(arr), Ok(c)); - let mut longer = [0xff; 5]; // 0xff is never valid - longer[..len].copy_from_slice(reference); - assert_eq!(char::from_utf8_slice_start(reference), Ok((c,len))); - assert_eq!(char::from_utf8_slice_start(&longer), Ok((c,len))); - assert_eq!(Utf8Char::from_slice_start(reference), Ok((u8c,len))); - assert_eq!(Utf8Char::from_slice_start(&longer), Ok((u8c,len))); - for other in &mut longer[len..] {*other = b'?'} - assert_eq!(Utf8Char::from_str(str_), Ok(u8c)); - assert_eq!(Utf8Char::from_str_start(str_), Ok((u8c,len))); - assert_eq!(Utf8Char::from_str_start(str::from_utf8(&longer).unwrap()), Ok((u8c,len))); - unsafe { - // Hopefully make bugs easier to catch by making reads into unallocated memory by filling - // a jemalloc bin. See table on http://jemalloc.net/jemalloc.3.html for bin sizes. - // I have no idea whether this works. - let mut boxed = Box::new([0xffu8; 16]); - let start = boxed.len()-len; // reach the end - boxed[start..].copy_from_slice(reference); - let slice = &boxed[start..start]; // length of slice should be ignored. - assert_eq!(Utf8Char::from_slice_start_unchecked(slice), (u8c,len)); - } - assert_eq!(&Vec::<u8>::from_iter(Some(u8c))[..], reference); - assert_eq!(&String::from_iter(Some(u8c))[..], str_); - assert_eq!(format!("{:?}", u8c), format!("{:?}", c)); - assert_eq!(format!("{}", u8c), format!("{}", c)); - assert_eq!(u8c.is_ascii(), c.is_ascii()); - assert_eq!(u8c.to_ascii_lowercase().to_char(), c.to_ascii_lowercase()); - assert_eq!(u8c.to_ascii_uppercase().to_char(), c.to_ascii_uppercase()); - - // UTF-16 - let mut buf = [0; 2]; - let reference = c.encode_utf16(&mut buf[..]); - let len = reference.len(); - assert_eq!(len, utf16_len); - assert_eq!(reference[0].utf16_needs_extra_unit(), Ok(len==2)); - assert_eq!(reference[0].is_utf16_leading_surrogate(), len==2); - assert_eq!(u16c.as_ref(), reference); - let mut longer = [0; 3]; - longer[..len].copy_from_slice(reference); - assert_eq!(char::from_utf16_slice_start(reference), Ok((c,len))); - assert_eq!(char::from_utf16_slice_start(&longer), Ok((c,len))); - assert_eq!(Utf16Char::from_slice_start(reference), Ok((u16c,len))); - assert_eq!(Utf16Char::from_slice_start(&longer), Ok((u16c,len))); - assert_eq!(Utf16Char::from_str(&as_str), Ok(u16c)); - as_str.push(c); - assert_eq!(Utf16Char::from_str_start(&as_str), Ok((u16c,utf8_len))); - unsafe { - // Hopefully make bugs easier to catch by making reads into unallocated memory by filling - // a jemalloc bin. See table on http://jemalloc.net/jemalloc.3.html for bin sizes. - // I have no idea whether this works. - let mut boxed = Box::new([0u16; 8]); - let start = boxed.len()-len; // reach the end - boxed[start..].copy_from_slice(reference); - let slice = &boxed[start..start]; // length of slice should be ignored. - assert_eq!(Utf16Char::from_slice_start_unchecked(slice), (u16c,len)); - } - let array = c.to_utf16_array(); - let tuple = c.to_utf16_tuple(); - assert_eq!(&array[..reference.len()], reference); - assert_eq!(tuple, (reference[0],reference.get(1).cloned())); - assert_eq!(char::from_utf16_array(array), Ok(c)); - assert_eq!(char::from_utf16_tuple(tuple), Ok(c)); - assert_eq!(c.to_utf16().to_char(), c); - assert_eq!(&Vec::<u16>::from_iter(Some(u16c))[..], reference); - assert_eq!(format!("{:?}", u16c), format!("{:?}", c)); - assert_eq!(format!("{}", u16c), format!("{}", c)); - assert_eq!(u16c.is_ascii(), c.is_ascii()); - assert_eq!(u16c.to_ascii_lowercase().to_char(), c.to_ascii_lowercase()); - assert_eq!(u16c.to_ascii_uppercase().to_char(), c.to_ascii_uppercase()); -} - - -#[test] -fn edges_middle() { - for &c in &EDGES_AND_BETWEEN { - test(c); - } -} - - -#[test] -#[ignore] -fn all() { - for cp in std::iter::Iterator::chain(0..0xd800, 0xe000..0x110000) { - let c = char::from_u32(cp).expect("not a valid char"); - test(c); - } -} |